You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:32 UTC
[01/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Repository: incubator-systemml
Updated Branches:
refs/heads/master a4264859d -> 7cd7dc2be
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
deleted file mode 100644
index 1c5024c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-
-
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-
-
-/**
- *
- *
- */
-public class RuntimePiggybacking
-{
-
- public enum PiggybackingType {
- TIME_BASED_SEQUENTIAL,
- UTIL_TIME_BASED_PARALLEL,
- UTIL_DECAY_BASED_PARALLEL,
- }
-
- private static PiggybackingType DEFAULT_WORKER_TYPE = PiggybackingType.UTIL_DECAY_BASED_PARALLEL;
-
- private static boolean _active = false;
- private static IDSequence _idSeq = null;
- private static PiggybackingWorker _worker = null;
-
- //mr instruction pool
- private static HashMap<JobType, LinkedList<Long>> _pool = null;
- private static HashMap<Long, MRJobInstruction> _jobs = null;
-
-
- //static initialization of piggybacking server
- static
- {
- //initialize mr-job instruction pool
- _pool = new HashMap<JobType, LinkedList<Long>>();
- _jobs = new HashMap<Long, MRJobInstruction>();
-
- //init id sequence
- _idSeq = new IDSequence();
- }
-
-
- /////////////////////////////////////////////////
- // public interface to runtime piggybacking
- ///////
-
- /**
- *
- * @return
- */
- public static boolean isActive()
- {
- return _active;
- }
-
- /**
- *
- * @param type
- * @param par
- * @throws DMLRuntimeException
- */
- public static void start( int par )
- throws DMLRuntimeException
- {
- start( DEFAULT_WORKER_TYPE, par );
- }
-
- /**
- * @throws DMLRuntimeException
- *
- */
- public static void start( PiggybackingType type, int par )
- throws DMLRuntimeException
- {
- //activate piggybacking server
- _active = true;
-
- //init job merge/submission worker
- switch( type )
- {
- case TIME_BASED_SEQUENTIAL:
- _worker = new PiggybackingWorkerTimeSequential();
- break;
- case UTIL_TIME_BASED_PARALLEL:
- _worker = new PiggybackingWorkerUtilTimeParallel(par);
- break;
- case UTIL_DECAY_BASED_PARALLEL:
- _worker = new PiggybackingWorkerUtilDecayParallel(par);
- break;
- default:
- throw new DMLRuntimeException("Unsupported runtime piggybacking type: "+type);
- }
-
- //start worker
- _worker.start();
- }
-
- /**
- *
- * @throws DMLRuntimeException
- */
- public static void stop()
- throws DMLRuntimeException
- {
- try
- {
- //deactivate piggybacking server
- _active = false;
-
- //cleanup merge/submission worker
- _worker.setStopped();
- _worker.join();
- _worker = null;
- }
- catch(InterruptedException ex)
- {
- throw new DMLRuntimeException("Failed to stop runtime piggybacking server.", ex);
- }
- }
-
- /**
- *
- * @param inst
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- public static JobReturn submitJob(MRJobInstruction inst)
- throws DMLRuntimeException
- {
- JobReturn ret = null;
-
- try
- {
- //step 1: obtain job id
- long id = _idSeq.getNextID();
-
- //step 2: append mr job to global pool
- synchronized( _pool )
- {
- //maintain job-type partitioned instruction pool
- if( !_pool.containsKey(inst.getJobType()) )
- _pool.put(inst.getJobType(), new LinkedList<Long>());
- _pool.get(inst.getJobType()).add( id );
-
- //add actual mr job instruction
- _jobs.put(id, inst);
- }
-
- //step 3: wait for finished job
- ret = _worker.getJobResult( id );
-
- if( !ret.successful )
- throw new DMLRuntimeException("Failed to run MR job via runtime piggybacking - job unsuccessful:\n"+inst.toString());
- }
- catch(InterruptedException ex)
- {
- throw new DMLRuntimeException("Failed to submit MR job to runtime piggybacking server.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @param type
- * @return
- */
- public static boolean isSupportedJobType( JobType type )
- {
- // reblock and datagen apply as well but this would limit the recompilation
- // potential of job-specific recompilation hooks due all-or-nothing semantics
-
- return ( type == JobType.GMR
- || type == JobType.CM_COV
- || type == JobType.GROUPED_AGG
- || type == JobType.REBLOCK );
- }
-
- /**
- * Gets a working set of MR job instructions out of the global pool.
- * All returned instructions are guaranteed to have the same job type
- * but are not necessarily mergable. This method returns the largest
- * instruction set currently in the pool.
- *
- * @return
- */
- protected static LinkedList<Pair<Long,MRJobInstruction>> getMaxWorkingSet()
- {
- LinkedList<Pair<Long,MRJobInstruction>> ret = null;
-
- synchronized( _pool )
- {
- //determine job type with max number of
- JobType currType = null;
- int currLength = 0;
- for( JobType jt : _pool.keySet() ){
- LinkedList<Long> tmp = _pool.get(jt);
- if( tmp!=null && currLength<tmp.size() ){
- currLength = tmp.size();
- currType = jt;
- }
- }
-
- //indicate empty pool if necessary
- if( currType==null )
- return null;
-
- //create working set and remove from pool
- ret = new LinkedList<Pair<Long,MRJobInstruction>>();
- LinkedList<Long> tmp = _pool.remove(currType);
- for( Long id : tmp )
- ret.add( new Pair<Long, MRJobInstruction>(id,_jobs.get(id)) );
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public static boolean isEmptyJobPool()
- {
- return _pool.isEmpty();
- }
-}
[36/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/GDFEnumOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/GDFEnumOptimizer.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/GDFEnumOptimizer.java
deleted file mode 100644
index d5d005b..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/GDFEnumOptimizer.java
+++ /dev/null
@@ -1,603 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.cost.CostEstimationWrapper;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFLoopNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.MismatchHeuristicFactory;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-
-/**
- * Global data flow optimization via enumeration-based optimizer (dynamic programming).
- *
- *
- * ADDITIONAL PERFORMANCE OPT (once everything is completely working)
- * TODO cache for interesting properties
- * TODO partial runtime plan generation
- * TODO partial runtime plan costing
- *
- */
-public class GDFEnumOptimizer extends GlobalOptimizer
-{
-
- private static final Log LOG = LogFactory.getLog(GDFEnumOptimizer.class);
-
- //internal configuration parameters
- //note: that branch and bound pruning is invalid if we cost entire programs
- private static final boolean BRANCH_AND_BOUND_PRUNING = true;
- private static final boolean PREFERRED_PLAN_SELECTION = true;
- private static final boolean COST_FULL_PROGRAMS = false;
- private static final boolean ENUM_CP_BLOCKSIZES = false;
- private static final MismatchHeuristicType DEFAULT_MISMATCH_HEURISTIC = MismatchHeuristicType.FIRST;
-
- //internal configuration parameters
- private static final int[] BLOCK_SIZES = new int[]{ 1024,//1 * DMLTranslator.DMLBlockSize,
- 2048,//2 * DMLTranslator.DMLBlockSize,
- 4096};//4 * DMLTranslator.DMLBlockSize};
- private static final double BRANCH_AND_BOUND_REL_THRES = Math.pow(10, -5);
- //private static final int[] REPLICATION_FACTORS = new int[]{1,3,5};
-
- private MemoStructure _memo = null; //plan memoization table
- private static GDFMismatchHeuristic _resolve = null;
- private static long _enumeratedPlans = 0;
- private static long _prunedInvalidPlans = 0;
- private static long _prunedSuboptimalPlans = 0;
- private static long _compiledPlans = 0;
- private static long _costedPlans = 0;
- private static long _planMismatches = 0;
-
-
- public GDFEnumOptimizer( )
- throws DMLRuntimeException
- {
- //init internal memo structure
- _memo = new MemoStructure();
-
- //init mismatch heuristic
- _resolve = MismatchHeuristicFactory.createMismatchHeuristic(
- DEFAULT_MISMATCH_HEURISTIC);
- }
-
- @Override
- public GDFGraph optimize(GDFGraph gdfgraph, Summary summary)
- throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException, LopsException
- {
- Timing time = new Timing(true);
-
- Program prog = gdfgraph.getRuntimeProgram();
- ExecutionContext ec = ExecutionContextFactory.createContext(prog);
- ArrayList<GDFNode> roots = gdfgraph.getGraphRootNodes();
-
- //Step 1: baseline costing for branch and bound costs
- double initCosts = Double.MAX_VALUE;
- if( BRANCH_AND_BOUND_PRUNING ) {
- initCosts = CostEstimationWrapper.getTimeEstimate(prog, ec);
- initCosts = initCosts * (1+BRANCH_AND_BOUND_REL_THRES);
- }
-
- //Step 2: dynamic programming plan generation
- //(finally, pick optimal root plans over all interesting property sets)
- ArrayList<Plan> rootPlans = new ArrayList<Plan>();
- for( GDFNode node : roots ) {
- PlanSet ps = enumOpt(node, _memo, initCosts);
- Plan optPlan = ps.getPlanWithMinCosts();
- rootPlans.add( optPlan );
- }
- long enumPlanMismatch = getPlanMismatches();
-
- //check for final containment of independent roots and pick optimal
- HashMap<Long, Plan> memo = new HashMap<Long,Plan>();
- resetPlanMismatches();
- for( Plan p : rootPlans )
- rSetRuntimePlanConfig(p, memo);
- long finalPlanMismatch = getPlanMismatches();
-
- //generate final runtime plan (w/ optimal config)
- Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
-
- ec = ExecutionContextFactory.createContext(prog);
- double optCosts = CostEstimationWrapper.getTimeEstimate(prog, ec);
-
- //maintain optimization summary statistics
- summary.setCostsInitial( initCosts );
- summary.setCostsOptimal( optCosts );
- summary.setNumEnumPlans( _enumeratedPlans );
- summary.setNumPrunedInvalidPlans( _prunedInvalidPlans );
- summary.setNumPrunedSuboptPlans( _prunedSuboptimalPlans );
- summary.setNumCompiledPlans( _compiledPlans );
- summary.setNumCostedPlans( _costedPlans );
- summary.setNumEnumPlanMismatch( enumPlanMismatch );
- summary.setNumFinalPlanMismatch( finalPlanMismatch );
- summary.setTimeOptim( time.stop() );
-
- return gdfgraph;
- }
-
- /**
- * Core dynamic programming enumeration algorithm
- * for global data flow optimization.
- *
- * @param node
- * @param maxCosts
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static PlanSet enumOpt( GDFNode node, MemoStructure memo, double maxCosts )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //memoization of already enumerated subgraphs
- if( memo.constainsEntry(node) )
- return memo.getEntry(node);
-
- //enumerate node plans
- PlanSet P = enumNodePlans( node, memo, maxCosts );
- //System.out.println("Plans after enumNodePlan:\n"+P.toString());
-
- //combine local node plan with optimal child plans
- for( GDFNode c : node.getInputs() )
- {
- //recursive optimization
- PlanSet Pc = enumOpt( c, memo, maxCosts );
- if( c instanceof GDFLoopNode )
- Pc = Pc.selectChild( node );
-
- //combine parent-child plans
- P = P.crossProductChild(Pc);
- _enumeratedPlans += P.size();
-
- //prune invalid plans
- pruneInvalidPlans( P );
- }
-
- //prune suboptimal plans
- pruneSuboptimalPlans( P, maxCosts );
-
- //memoization of created entries
- memo.putEntry(node, P);
-
- return P;
- }
-
- /**
- *
- * @param node
- * @param memo
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- private static PlanSet enumNodePlans( GDFNode node, MemoStructure memo, double maxCosts )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Plan> plans = new ArrayList<Plan>();
- ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //ENUMERATE HOP PLANS
- // CASE 1: core hop enumeration (other than persistent/transient read/write)
- if( node.getNodeType() == NodeType.HOP_NODE && !(node.getHop() instanceof DataOp ) )
- {
- //core rewrite enumeration for cp and mr
- enumHopNodePlans(node, plans);
- }
- //CASE 2: dataop hop enumeration
- else if( node.getHop() instanceof DataOp )
- {
- DataOp dhop = (DataOp)node.getHop();
-
- if( dhop.getDataOpType()==DataOpTypes.PERSISTENTREAD )
- {
- //for persistent read the interesting properties are fixed by the input
- //but we can decide on output properties
- ExecType et = (dhop.getMemEstimate()>OptimizerUtils.getLocalMemBudget()
- || HopRewriteUtils.alwaysRequiresReblock(dhop)) ?
- CLUSTER : ExecType.CP;
-
- int[] blocksizes = (et == CLUSTER) ? BLOCK_SIZES : new int[]{BLOCK_SIZES[0]};
- for( Integer bs : blocksizes )
- {
- RewriteConfig rcmr = new RewriteConfig(et, bs, FileFormatTypes.BINARY);
- InterestingProperties ipsmr = rcmr.deriveInterestingProperties();
- Plan mrplan = new Plan(node, ipsmr, rcmr, null);
- plans.add( mrplan );
- }
- }
- else if( dhop.getDataOpType()==DataOpTypes.PERSISTENTWRITE )
- {
- //for persistent write the interesting properties are fixed by the given
- //write specification
- ExecType et = (dhop.getMemEstimate()>OptimizerUtils.getLocalMemBudget()) ?
- CLUSTER : ExecType.CP;
-
- RewriteConfig rcmr = new RewriteConfig(et, (int)dhop.getRowsInBlock(), dhop.getInputFormatType());
- InterestingProperties ipsmr = rcmr.deriveInterestingProperties();
- Plan mrplan = new Plan(node, ipsmr, rcmr, null);
- plans.add( mrplan );
- }
- else if( dhop.getDataOpType()==DataOpTypes.TRANSIENTREAD
- || dhop.getDataOpType()==DataOpTypes.TRANSIENTWRITE)
- {
- //note: full enumeration for transient read and write; otherwise the properties
- //of these hops are never set because pass-through plans refer to different hops
- enumHopNodePlans(node, plans);
- }
- }
- //ENUMERATE LOOP PLANS
- else if( node.getNodeType() == NodeType.LOOP_NODE )
- {
- //TODO consistency checks inputs and outputs (updated vars)
-
- GDFLoopNode lnode = (GDFLoopNode) node;
-
- //step 0: recursive call optimize on inputs
- //no additional pruning (validity, optimality) required
- for( GDFNode in : lnode.getLoopInputs().values() )
- enumOpt(in, memo, maxCosts);
-
- //step 1: enumerate loop plan, incl partitioning/checkpoints/reblock for inputs
- RewriteConfig rc = new RewriteConfig(ExecType.CP, -1, null);
- InterestingProperties ips = rc.deriveInterestingProperties();
- Plan lplan = new Plan(node, ips, rc, null);
- plans.add( lplan );
-
- //step 2: recursive call optimize on predicate
- //(predicate might be null if single variable)
- if( lnode.getLoopPredicate() != null )
- enumOpt(lnode.getLoopPredicate(), memo, maxCosts);
-
- //step 3: recursive call optimize on outputs
- //(return union of all output plans, later selected by output var)
- PlanSet Pout = new PlanSet();
- for( GDFNode out : lnode.getLoopOutputs().values() )
- Pout = Pout.union( enumOpt(out, memo, maxCosts) );
- plans.addAll(Pout.getPlans());
-
- //note: global pruning later done when returning to enumOpt
- //for the entire loop node
- }
- //CREATE DUMMY CROSSBLOCK PLAN
- else if( node.getNodeType() == NodeType.CROSS_BLOCK_NODE )
- {
- //do nothing (leads to pass-through on crossProductChild)
- }
-
- return new PlanSet(plans);
- }
-
- /**
- *
- * @param node
- * @param plans
- */
- private static void enumHopNodePlans(GDFNode node, ArrayList<Plan> plans)
- {
- ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //create cp plan, if allowed (note: most interesting properties are irrelevant for CP)
- if( node.getHop().getMemEstimate() < OptimizerUtils.getLocalMemBudget() ) {
- int[] bstmp = ENUM_CP_BLOCKSIZES ? BLOCK_SIZES : new int[]{BLOCK_SIZES[0]};
- for( Integer bs : bstmp ) {
- RewriteConfig rccp = new RewriteConfig(ExecType.CP, bs, FileFormatTypes.BINARY);
- InterestingProperties ipscp = rccp.deriveInterestingProperties();
- Plan cpplan = new Plan(node, ipscp, rccp, null);
- plans.add( cpplan );
- }
- }
-
- //create mr plans, if required
- if( node.requiresMREnumeration() ) {
- for( Integer bs : BLOCK_SIZES )
- {
- RewriteConfig rcmr = new RewriteConfig(CLUSTER, bs, FileFormatTypes.BINARY);
- InterestingProperties ipsmr = rcmr.deriveInterestingProperties();
- Plan mrplan = new Plan(node, ipsmr, rcmr, null);
- plans.add( mrplan );
- }
- }
- }
-
- /**
- *
- * @param plans
- */
- private static void pruneInvalidPlans( PlanSet plans )
- {
- ArrayList<Plan> valid = new ArrayList<Plan>();
-
- //check each plan in planset for validity
- for( Plan plan : plans.getPlans() )
- {
- //a) check matching blocksizes if operation in MR
- if( !plan.checkValidBlocksizesInMR() ) {
- //System.out.println("pruned invalid blocksize mr op");
- continue;
- }
-
- //b) check matching blocksizes of tread/twrite pairs
- if( !plan.checkValidBlocksizesTRead() ) {
- //System.out.println("pruned invalid blocksize tread");
- continue;
- }
-
- //c) check valid format in MR
- if( !plan.checkValidFormatInMR() ) {
- //System.out.println("pruned invalid format: "+plan.getNode().getHop().getClass());
- continue;
- }
-
- //d) check valid execution type per hop (e.g., function, reblock)
- if( !plan.checkValidExecutionType() ) {
- //System.out.println("pruned invalid execution type: "+plan.getNode().getHop().getClass());
- continue;
- }
-
- valid.add( plan );
- }
-
- //debug output
- int sizeBefore = plans.size();
- int sizeAfter = valid.size();
- _prunedInvalidPlans += (sizeBefore-sizeAfter);
- LOG.debug("Pruned invalid plans: "+sizeBefore+" --> "+sizeAfter);
-
- plans.setPlans( valid );
- }
-
- /**
- *
- * @param plans
- * @param maxCosts
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private static void pruneSuboptimalPlans( PlanSet plans, double maxCosts )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //costing of all plans incl containment check
- for( Plan p : plans.getPlans() ) {
- p.setCosts( costRuntimePlan(p) );
- }
-
- //build and probe for optimal plans (hash-groupby on IPC, min costs)
- HashMap<InterestingProperties, Plan> probeMap = new HashMap<InterestingProperties, Plan>();
- for( Plan p : plans.getPlans() )
- {
- //max cost pruning filter (branch-and-bound)
- if( BRANCH_AND_BOUND_PRUNING && p.getCosts() > maxCosts ) {
- continue;
- }
-
- //plan cost per IPS pruning filter (allow smaller or equal costs)
- Plan best = probeMap.get(p.getInterestingProperties());
- if( best!=null && p.getCosts() > best.getCosts() ) {
- continue;
- }
-
- //non-preferred plan pruning filter (allow smaller cost or equal cost and preferred plan)
- if( PREFERRED_PLAN_SELECTION && best!=null &&
- p.getCosts() == best.getCosts() && !p.isPreferredPlan() ) {
- continue;
- }
-
- //add plan as best per IPS
- probeMap.put(p.getInterestingProperties(), p);
-
- }
-
- //copy over plans per IPC into one plan set
- ArrayList<Plan> optimal = new ArrayList<Plan>(probeMap.values());
-
- int sizeBefore = plans.size();
- int sizeAfter = optimal.size();
- _prunedSuboptimalPlans += (sizeBefore-sizeAfter);
- LOG.debug("Pruned suboptimal plans: "+sizeBefore+" --> "+sizeAfter);
-
- plans.setPlans(optimal);
- }
-
- /**
- *
- * @param p
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private static double costRuntimePlan(Plan p)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- Program prog = p.getNode().getProgram();
- if( prog == null )
- throw new DMLRuntimeException("Program not available for runtime plan costing.");
-
- //put data flow configuration into program
- rSetRuntimePlanConfig(p, new HashMap<Long,Plan>());
-
- double costs = -1;
- if( COST_FULL_PROGRAMS ||
- (p.getNode().getHop()==null || p.getNode().getProgramBlock()==null) )
- {
- //recompile entire runtime program
- Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
- _compiledPlans++;
-
- //cost entire runtime program
- ExecutionContext ec = ExecutionContextFactory.createContext(prog);
- costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
- }
- else
- {
- Hop currentHop = p.getNode().getHop();
- ProgramBlock pb = p.getNode().getProgramBlock();
-
- try
- {
- //keep the old dag roots
- ArrayList<Hop> oldRoots = pb.getStatementBlock().get_hops();
- Hop tmpHop = null;
- if( !(currentHop instanceof DataOp && ((DataOp)currentHop).isWrite()) ){
- ArrayList<Hop> newRoots = new ArrayList<Hop>();
- tmpHop = new DataOp("_tmp", currentHop.getDataType(), currentHop.getValueType(), currentHop, DataOpTypes.TRANSIENTWRITE, "tmp");
- tmpHop.setVisited(VisitStatus.DONE); //ensure recursive visitstatus reset on recompile
- newRoots.add(tmpHop);
- pb.getStatementBlock().set_hops(newRoots);
- }
-
- //recompile modified runtime program
- Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
- _compiledPlans++;
-
- //cost partial runtime program up to current hop
- ExecutionContext ec = ExecutionContextFactory.createContext(prog);
- costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
-
- //restore original hop dag
- if( tmpHop !=null )
- HopRewriteUtils.removeChildReference(tmpHop, currentHop);
- pb.getStatementBlock().set_hops(oldRoots);
- }
- catch(HopsException ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
- //release forced data flow configuration from program
- rResetRuntimePlanConfig(p, new HashMap<Long,Plan>());
- _costedPlans++;
-
- return costs;
- }
-
- private static void rSetRuntimePlanConfig( Plan p, HashMap<Long, Plan> memo )
- {
- ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //basic memoization including containment check
- if( memo.containsKey(p.getNode().getID()) ) {
- Plan pmemo = memo.get(p.getNode().getID());
- if( !p.getInterestingProperties().equals( pmemo.getInterestingProperties()) )
- {
- //replace plan in memo with new plan
- //TODO this would require additional cleanup in special cases
- if( _resolve.resolveMismatch(pmemo.getRewriteConfig(), p.getRewriteConfig()) )
- memo.put(p.getNode().getID(), p);
- //logging of encounter plan mismatch
- LOG.warn("Configuration mismatch on shared node ("+p.getNode().getHop().getHopID()+"). Falling back to heuristic '"+_resolve.getName()+"'.");
- LOG.warn(p.getInterestingProperties().toString());
- LOG.warn(memo.get(p.getNode().getID()).getInterestingProperties());
- _planMismatches++;
- return;
- }
- }
-
- //set plan configuration
- Hop hop = p.getNode().getHop();
- if( hop!=null )
- {
- RewriteConfig rc = p.getRewriteConfig();
- //set exec type
- hop.setForcedExecType(rc.getExecType());
- //set blocksizes and reblock
- hop.setRowsInBlock(rc.getBlockSize());
- hop.setColsInBlock(rc.getBlockSize());
- if( rc.getExecType()==CLUSTER ) //after blocksize update
- {
- //TODO double check dataop condition - side effect from plan validity
- boolean reblock = HopRewriteUtils.alwaysRequiresReblock(hop) ||
- (hop.hasMatrixInputWithDifferentBlocksizes() && !(hop instanceof DataOp));
- hop.setRequiresReblock(reblock);
- }
- else
- hop.setRequiresReblock(false);
- }
-
- //process childs
- if( p.getChilds() != null )
- for( Plan c : p.getChilds() )
- rSetRuntimePlanConfig(c, memo);
-
- //memoization (mark as processed)
- memo.put(p.getNode().getID(), p);
- }
-
- private static void rResetRuntimePlanConfig( Plan p, HashMap<Long, Plan> memo )
- {
- //basic memoization including containment check
- if( memo.containsKey(p.getNode().getID()) ) {
- return;
- }
-
- //release forced plan configuration
- Hop hop = p.getNode().getHop();
- if( hop!=null )
- {
- hop.setForcedExecType(null);
- hop.setRowsInBlock(DMLTranslator.DMLBlockSize);
- hop.setColsInBlock(DMLTranslator.DMLBlockSize);
- if( !HopRewriteUtils.alwaysRequiresReblock(hop) ) {
- hop.setRequiresReblock(false);
- }
- }
-
- //process childs
- if( p.getChilds() != null )
- for( Plan c : p.getChilds() )
- rResetRuntimePlanConfig(c, memo);
-
- //memoization (mark as processed)
- memo.put(p.getNode().getID(), p);
- }
-
- private static long getPlanMismatches(){
- return _planMismatches;
- }
-
- private static void resetPlanMismatches(){
- _planMismatches = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizer.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizer.java
deleted file mode 100644
index f48ad33..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-
-/**
- * Super class for all optimizers (e.g., transformation-based, and enumeration-based)
- *
- */
-public abstract class GlobalOptimizer
-{
-
- /**
- * Core optimizer call, to be implemented by an instance of a global
- * data flow optimizer.
- *
- * @param prog
- * @param rtprog
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws
- */
- public abstract GDFGraph optimize( GDFGraph gdfgraph, Summary summary )
- throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException, LopsException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizerWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizerWrapper.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizerWrapper.java
deleted file mode 100644
index be15d07..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/GlobalOptimizerWrapper.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GraphBuilder;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.utils.Explain;
-
-/**
- * Main entry point for Global Data Flow Optimization. It is intended to be invoked after
- * the initial runtime program was created (and thus with constructed hops and lops).
- *
- *
- */
-public class GlobalOptimizerWrapper
-{
-
- private static final Log LOG = LogFactory.getLog(GlobalOptimizerWrapper.class);
- private static final boolean LDEBUG = true; //local debug flag
-
- //supported optimizers
- public enum GlobalOptimizerType{
- ENUMERATE_DP,
- TRANSFORM,
- }
-
- //internal parameters
- private static final GlobalOptimizerType OPTIM = GlobalOptimizerType.ENUMERATE_DP;
-
- static
- {
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.globalopt")
- .setLevel((Level) Level.DEBUG);
- }
- }
-
- /**
- *
- * @param prog
- * @param rtprog
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- */
- public static Program optimizeProgram(DMLProgram prog, Program rtprog)
- throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException, LopsException
- {
- LOG.debug("Starting global data flow optimization.");
- Timing time = new Timing(true);
-
- //create optimizer instance
- GlobalOptimizer optimizer = createGlobalOptimizer( OPTIM );
-
- //create global data flow graph
- Summary summary = new Summary();
- GDFGraph graph = GraphBuilder.constructGlobalDataFlowGraph(rtprog, summary);
- if( LOG.isDebugEnabled() ) {
- LOG.debug("EXPLAIN GDFGraph:\n" + Explain.explainGDFNodes(graph.getGraphRootNodes(),1));
- }
-
- //core global data flow optimization
- graph = optimizer.optimize(graph, summary);
-
- //get the final runtime program
- rtprog = graph.getRuntimeProgram();
-
- //print global optimizer summary
- LOG.info( summary );
-
- LOG.debug("Finished global data flow optimization in " + time.stop() + " ms.");
- return rtprog;
- }
-
- /**
- *
- * @param type
- * @param graphCreator
- * @return
- * @throws HopsException
- * @throws DMLRuntimeException
- */
- private static GlobalOptimizer createGlobalOptimizer( GlobalOptimizerType type )
- throws HopsException, DMLRuntimeException
- {
- GlobalOptimizer optimizer = null;
-
- switch( type )
- {
- case ENUMERATE_DP:
- optimizer = new GDFEnumOptimizer();
- break;
-
- //case TRANSFORM:
- // optimizer = new GlobalTransformationOptimizer(Strategy.CANONICAL);
- // ((GlobalTransformationOptimizer)optimizer).addRule(new BlockSizeRule());
- // break;
-
- default:
- throw new HopsException("Unsupported global optimizer type: "+type+".");
- }
-
- return optimizer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/InterestingProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/InterestingProperties.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/InterestingProperties.java
deleted file mode 100644
index 7f58b09..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/InterestingProperties.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import com.google.common.base.Objects;
-
-/**
- * An instance of this class represents one 'interesting property set' defined by the instances
- * of all interesting properties. We do not use objects per interesting property in order to
- * (1) simplify the creation and interesting property sets, and (2) prevent excessive object
- * creation and garbage collection.
- *
- */
-public class InterestingProperties
-{
-
- public enum Location {
- MEM,
- HDFS_CACHE,
- HDFS,
- }
-
- public enum Format {
- ANY,
- BINARY_BLOCK,
- BINARY_CELL,
- TEXT_CELL,
- TEXT_MM,
- TEXT_CSV,
- }
-
- public enum Partitioning {
- NONE,
- ROW_WISE,
- COL_WISE,
- //ROW_BLOCK_WISE,
- //COL_BLOCK_WISE,
- }
-
- //supported interesting properties
- private int _blocksize = -1; //-1 for any
- private Format _format = null;
- private Location _location = null;
- private Partitioning _pformat = null;
- private int _replication = -1;
- private boolean _emptyblocks = false;
-
-
- public InterestingProperties( int blocksize, Format format, Location location, Partitioning pformat, int replication, boolean emptyblocks )
- {
- _blocksize = blocksize;
- _format = format;
- _location = location;
- _pformat = pformat;
- _replication = replication;
- _emptyblocks = emptyblocks;
- }
-
- public InterestingProperties( InterestingProperties that )
- {
- _blocksize = that._blocksize;
- _format = that._format;
- _location = that._location;
- _pformat = that._pformat;
- _replication = that._replication;
- _emptyblocks = that._emptyblocks;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if( !(o instanceof InterestingProperties) )
- return false;
-
- InterestingProperties that = (InterestingProperties)o;
- return ( _blocksize == that._blocksize
- && _format == that._format
- && _location == that._location
- && _pformat == that._pformat
- && _replication == that._replication
- && _emptyblocks == that._emptyblocks );
- }
-
- @Override
- public int hashCode()
- {
- //relies on google's guava library
- return Objects.hashCode(
- _blocksize,
- (_format!=null)?_format.ordinal():-1,
- (_location!=null)?_location.ordinal():-1,
- (_pformat!=null)?_pformat.ordinal():-1,
- _replication,
- _emptyblocks
- );
- }
-
- @Override
- public String toString()
- {
- return "IPS[" + _blocksize + "," + _format + "," + _location + "," + _pformat + "," + _replication + "," + _emptyblocks + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/MemoStructure.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/MemoStructure.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/MemoStructure.java
deleted file mode 100644
index 0cdd576..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/MemoStructure.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-
-/**
- * This MemoStructure is the central location for storing enumerated plans (configurations) and serves
- * two purposes:
- *
- * 1) Plan Memoization: Due to the DAG structure (where a single node is reachable over alternative
- * paths), our top-down, recursive optimization procedure might visit an operator multiple times.
- * The memo structure memoizes and reuses already generated plans.
- *
- * 2) config set cache
- *
- * The internal structure is as follows:
- * Memo Structure := | LONG GDFNodeID | MemoEntry | *
- * Memo Entry := | InterestingPropertySet | DOUBLE COST | *
- *
- */
-public class MemoStructure
-{
-
- private HashMap<Long, PlanSet> _entries = null; //NODEID | PLANSET
- private HashMap<Long, Long> _nodeIDs = null; //NODEID | HOPID
-
- public MemoStructure()
- {
- _entries = new HashMap<Long, PlanSet>();
- _nodeIDs = new HashMap<Long, Long>();
- }
-
- ///////////////////////////////////////////////////
- // basic access to memo structure entries
-
- public boolean constainsEntry( GDFNode node )
- {
- return _entries.containsKey( node.getID() );
- }
-
- public void putEntry( GDFNode node, PlanSet entry )
- {
- _entries.put( node.getID(), entry );
- if( node.getHop()!=null )
- _nodeIDs.put( node.getID(), node.getHop().getHopID() );
- else
- _nodeIDs.put( node.getID(), -1L );
- }
-
- public PlanSet getEntry( GDFNode node )
- {
- return _entries.get( node.getID()) ;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("------------------------------------\n");
- sb.append("MEMO STRUCTURE (gdfnodeid, plans): \n");
- sb.append("------------------------------------\n");
-
- for( Entry<Long, PlanSet> e : _entries.entrySet() )
- {
- sb.append("------------------------------------\n");
- sb.append("Node "+e.getKey()+" (hop "+_nodeIDs.get(e.getKey())+"):\n");
- for( Plan p : e.getValue().getPlans() ) {
- sb.append(p.toString());
- sb.append("\n");
- }
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/Plan.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/Plan.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/Plan.java
deleted file mode 100644
index aa9b29e..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/Plan.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-
-
-public class Plan
-{
- private static IDSequence _seqID = new IDSequence();
-
- private long _ID = -1;
- private GDFNode _node = null;
-
- private InterestingProperties _ips = null;
- private RewriteConfig _conf = null;
- private ArrayList<Plan> _childs = null;
- private double _costs = -1;
-
- public Plan(GDFNode node, InterestingProperties ips, RewriteConfig rc, ArrayList<Plan> childs)
- {
- _ID = _seqID.getNextID();
- _node = node;
- _ips = ips;
- _conf = rc;
- if( childs != null && !childs.isEmpty() )
- _childs = childs;
- else
- _childs = new ArrayList<Plan>();
- }
-
- public Plan( Plan p )
- {
- _ID = _seqID.getNextID();
- _node = p._node;
- _ips = new InterestingProperties(p._ips);
- _conf = new RewriteConfig(p._conf);
- _costs = p._costs;
-
- if( p._childs != null && !p._childs.isEmpty() )
- _childs = new ArrayList<Plan>(p._childs);
- else
- _childs = new ArrayList<Plan>();
- }
-
- public GDFNode getNode()
- {
- return _node;
- }
-
- public void addChild( Plan c )
- {
- _childs.add(c);
- }
-
- public ArrayList<Plan> getChilds()
- {
- return _childs;
- }
-
- public InterestingProperties getInterestingProperties()
- {
- return _ips;
- }
-
- public RewriteConfig getRewriteConfig()
- {
- return _conf;
- }
-
- public void setCosts( double costs )
- {
- _costs = costs;
- }
-
- public double getCosts()
- {
- return _costs;
- }
-
- /**
- * If operation is executed in MR, all input blocksizes need to match.
- * Note that the output blocksize can be different since we would add
- * additional reblocks after that operation.
- *
- * @return
- */
- public boolean checkValidBlocksizesInMR()
- {
- boolean ret = true;
- ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _conf.getExecType()==CLUSTER
- && _childs != null && _childs.size() > 1 )
- {
- int size0 = _childs.get(0)._conf.getBlockSize();
- if( size0 > 0 ) { //-1 compatible with everything
- for( Plan c : _childs )
- ret &= ( c._conf.getBlockSize() == size0
- ||c._conf.getBlockSize() <= 0 );
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public boolean checkValidBlocksizesTRead()
- {
- boolean ret = true;
-
- if( _node.getNodeType() == NodeType.HOP_NODE
- && _node.getHop() instanceof DataOp
- && ((DataOp)_node.getHop()).getDataOpType() == DataOpTypes.TRANSIENTREAD )
- {
- for( Plan c : _childs )
- ret &= ( _conf.getBlockSize() == c._conf.getBlockSize() );
- }
-
- if( _node.getNodeType() == NodeType.CROSS_BLOCK_NODE )
- {
- for( Plan c : _childs )
- ret &= ( _conf.getBlockSize() == c._conf.getBlockSize() );
- }
-
- return ret;
- }
-
- /**
- * If operation is executed in MR, only certain operations allow
- * all formats. In general, unary operations also allow for cell inputs.
- * TODO: check and test current format assumptions
- *
- * @param node
- * @return
- */
- public boolean checkValidFormatInMR()
- {
- boolean ret = true;
- ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _conf.getExecType()==CLUSTER )
- {
- if( _childs != null )
- for( Plan c : _childs )
- ret &= _node.isValidInputFormatForOperation(c._conf.getFormat());
- }
-
- return ret;
- }
-
- public boolean checkValidExecutionType()
- {
- boolean ret = true;
-
- ret &= !( _node.getHop() instanceof FunctionOp && _conf.getExecType()!=ExecType.CP );
- //unnecessary, because reblock now merged into base hop
- //ret &= !( _node.getHop() instanceof ReblockOp && _conf.getExecType()!=ExecType.MR );
-
- return ret;
- }
-
- /**
- * A plan is defined as preferred if its output interesting properties
- * match the interesting properties of all its matrix inputs.
- *
- * @return
- */
- public boolean isPreferredPlan()
- {
- boolean ret = true;
-
- for( Plan c : _childs )
- if( c.getNode().getDataType()==DataType.MATRIX )
- ret &= _ips.equals( c.getInterestingProperties() );
-
- return ret;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("PLAN("+_ID+") [");
- sb.append(_ips.toString());
- sb.append(",");
- sb.append(_conf.toString());
- sb.append(",{");
- for( Plan c : _childs ){
- sb.append(c._ID);
- sb.append(",");
- }
- sb.setLength(sb.length()-1);
- sb.append("},");
- sb.append(_costs);
- sb.append("]");
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/PlanSet.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/PlanSet.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/PlanSet.java
deleted file mode 100644
index cf1897d..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/PlanSet.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFCrossBlockNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-
-public class PlanSet
-{
-
- private ArrayList<Plan> _plans = null;
-
- public PlanSet()
- {
- _plans = new ArrayList<Plan>();
- }
-
- public PlanSet(ArrayList<Plan> plans)
- {
- _plans = plans;
- }
-
- public ArrayList<Plan> getPlans()
- {
- return _plans;
- }
-
- public void setPlans(ArrayList<Plan> plans)
- {
- _plans = plans;
- }
-
- public int size()
- {
- if( _plans == null )
- return 0;
-
- return _plans.size();
- }
-
- public boolean isEmpty()
- {
- if( _plans == null )
- return true;
-
- return _plans.isEmpty();
- }
-
- /**
- *
- * @param pc
- * @return
- */
- public PlanSet crossProductChild(PlanSet pc)
- {
- //check for empty child plan set (return current plan set)
- if( pc==null || pc.isEmpty() ) {
- return this;
- }
- //check for empty parent plan set (pass-through child)
- //(e.g., for crossblockop; this also implicitly reused costed runtime plans)
- if( _plans == null || _plans.isEmpty() ) {
- return pc;
- }
-
- ArrayList<Plan> Pnew = new ArrayList<Plan>();
-
- // create cross product of plansets between partial and child plans
- for( Plan p : _plans )
- for( Plan c : pc.getPlans() )
- {
- Plan pnew = new Plan(p);
- pnew.addChild( c );
- Pnew.add( pnew );
- }
-
- return new PlanSet(Pnew);
- }
-
- /**
- *
- * @param node
- * @return
- */
- public PlanSet selectChild( GDFNode node )
- {
- String varname = (node.getNodeType()==NodeType.HOP_NODE) ? node.getHop().getName() :
- ((GDFCrossBlockNode)node).getName();
-
- ArrayList<Plan> Pnew = new ArrayList<Plan>();
- for( Plan p : _plans )
- if( p.getNode().getHop()!=null
- &&p.getNode().getHop().getName().equals(varname) )
- {
- Pnew.add( p );
- }
-
- return new PlanSet(Pnew);
- }
-
- /**
- *
- * @param ps
- * @return
- */
- public PlanSet union( PlanSet ps )
- {
- ArrayList<Plan> Pnew = new ArrayList<Plan>(_plans);
- for( Plan p : ps._plans )
- Pnew.add( p );
-
- return new PlanSet(Pnew);
- }
-
- /**
- *
- * @return
- */
- public Plan getPlanWithMinCosts()
- {
- //init global optimal plan and costs
- double optCosts = Double.MAX_VALUE;
- Plan optPlan = null;
-
- //compare costs of all plans
- for( Plan p : _plans ) {
- if( p.getCosts() < optCosts ) {
- optCosts = p.getCosts();
- optPlan = p;
- }
- }
-
- return optPlan;
- }
-
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("PLAN_SET@"+super.hashCode()+":\n");
- for( Plan p : _plans ) {
- sb.append("--");
- sb.append( p.toString() );
- sb.append("\n");
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/RewriteConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/RewriteConfig.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/RewriteConfig.java
deleted file mode 100644
index a202a8a..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/RewriteConfig.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties.Location;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-
-
-
-/**
- * This RewriteConfig represents an instance configuration of a particular rewrite.
- *
- */
-public class RewriteConfig
-{
-
- /*
- public enum RewriteConfigType {
- BLOCK_SIZE,
- FORMAT_CHANGE,
- EXEC_TYPE,
- DATA_PARTITIONING,
- VECTORIZATION,
- REPLICATION_FACTOR
- }
- */
-
- private ExecType _rewriteSetExecType = null;
- private int _rewriteSetBlockSize = -1;
- private FileFormatTypes _rewriteFormat = null;
-
- public RewriteConfig( ExecType et, int bs, FileFormatTypes format )
- {
- _rewriteSetExecType = et;
- _rewriteSetBlockSize = bs;
- _rewriteFormat = format;
- }
-
- public RewriteConfig( RewriteConfig rc )
- {
- _rewriteSetExecType = rc._rewriteSetExecType;
- _rewriteSetBlockSize = rc._rewriteSetBlockSize;
- _rewriteFormat = rc._rewriteFormat;
- }
-
- public ExecType getExecType()
- {
- return _rewriteSetExecType;
- }
-
- public int getBlockSize()
- {
- return _rewriteSetBlockSize;
- }
-
- public FileFormatTypes getFormat()
- {
- return _rewriteFormat;
- }
-
- /**
- *
- * @return
- */
- public InterestingProperties deriveInterestingProperties()
- {
- int bs = _rewriteSetBlockSize;
- Location loc = (_rewriteSetExecType==ExecType.CP) ? Location.MEM : Location.HDFS;
-
- return new InterestingProperties(bs, null, loc, null, -1, true);
- }
-
-
- @Override
- public String toString()
- {
- return "RC["+_rewriteSetExecType+","+_rewriteSetBlockSize+"]";//+_type+"="+_value + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/Summary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/Summary.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/Summary.java
deleted file mode 100644
index 2814a61..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/Summary.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt;
-
-public class Summary
-{
-
- private double _costsInitial = -1;
- private double _costsOptimal = -1;
- private long _numEnumPlans = -1;
- private long _numPrunedInvalidPlans = -1;
- private long _numPrunedSuboptPlans = -1;
- private long _numCompiledPlans = -1;
- private long _numCostedPlans = -1;
- private long _numEnumPlanMismatch = -1;
- private long _numFinalPlanMismatch = -1;
- private double _timeGDFGraph = -1;
- private double _timeOptim = -1;
-
- public void setCostsInitial(double costsInitial) {
- _costsInitial = costsInitial;
- }
-
- public void setCostsOptimal(double costsOptimal) {
- _costsOptimal = costsOptimal;
- }
-
- public void setNumEnumPlans(long numEnumPlans) {
- _numEnumPlans = numEnumPlans;
- }
-
- public void setNumPrunedInvalidPlans(long numPrunedInvalidPlans) {
- _numPrunedInvalidPlans = numPrunedInvalidPlans;
- }
-
- public void setNumPrunedSuboptPlans(long numPrunedSuboptPlans) {
- _numPrunedSuboptPlans = numPrunedSuboptPlans;
- }
-
- public void setNumCompiledPlans(long numCompiledPlans) {
- _numCompiledPlans = numCompiledPlans;
- }
-
- public void setNumCostedPlans(long numCostedPlans) {
- _numCostedPlans = numCostedPlans;
- }
-
- public void setNumEnumPlanMismatch(long numEnumPlanMismatch) {
- _numEnumPlanMismatch = numEnumPlanMismatch;
- }
-
- public void setNumFinalPlanMismatch(long numFinalPlanMismatch) {
- _numFinalPlanMismatch = numFinalPlanMismatch;
- }
-
- public void setTimeGDFGraph(double timeGDFGraph) {
- _timeGDFGraph = timeGDFGraph;
- }
-
- public void setTimeOptim(double timeOptim) {
- _timeOptim = timeOptim;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append("\nGlobal Optimization Summary:\n");
- sb.append("-- costs of initial plan: "+_costsInitial +"\n");
- sb.append("-- costs of optimal plan: "+_costsOptimal +"\n");
- sb.append("-- # enumerated plans: "+_numEnumPlans +"\n");
- sb.append("-- # pruned invalid plans: "+_numPrunedInvalidPlans +"\n");
- sb.append("-- # pruned subopt plans: "+_numPrunedSuboptPlans +"\n");
- sb.append("-- # program compilations: "+_numCompiledPlans +"\n");
- sb.append("-- # program costings: "+_numCostedPlans +"\n");
- sb.append("-- # enum plan mismatch: "+_numEnumPlanMismatch +"\n");
- sb.append("-- # final plan mismatch: "+_numFinalPlanMismatch +"\n");
- sb.append("-- graph creation time: "+String.format("%.3f", _timeGDFGraph/1000)+" sec.\n");
- sb.append("-- optimization time: "+String.format("%.3f", _timeOptim/1000)+" sec.");
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
deleted file mode 100644
index 60d95ce..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-
-/**
- * Crossblock operators represent
- *
- */
-public class GDFCrossBlockNode extends GDFNode
-{
-
-
- public enum CrossBlockNodeType {
- PLAIN,
- MERGE,
- }
-
- private CrossBlockNodeType _cbtype = null;
- private String _name = null;
-
- /**
- * Constructor PLAIN crossblocknode
- *
- * @param hop
- * @param pb
- * @param input
- * @param name
- */
- public GDFCrossBlockNode( Hop hop, ProgramBlock pb, GDFNode input, String name )
- {
- super(hop, pb, null);
- _type = NodeType.CROSS_BLOCK_NODE;
- _inputs = new ArrayList<GDFNode>();
- _inputs.add( input );
-
- _cbtype = CrossBlockNodeType.PLAIN;
- _name = name;
- }
-
- /**
- * Constructor MERGE crossblocknode
- *
- * @param hop
- * @param pb
- * @param input1
- * @param input2
- * @param name
- */
- public GDFCrossBlockNode( Hop hop, ProgramBlock pb, GDFNode input1, GDFNode input2, String name )
- {
- super(hop, pb, null);
- _type = NodeType.CROSS_BLOCK_NODE;
- _inputs = new ArrayList<GDFNode>();
- _inputs.add( input1 );
- _inputs.add( input2 );
-
- _cbtype = CrossBlockNodeType.MERGE;
- _name = name;
- }
-
- public String getName()
- {
- return _name;
- }
-
- @Override
- public String explain(String deps)
- {
- String ldeps = (deps!=null) ? deps : "";
-
- return "CBNode "+ldeps+" ["+_name+", "+_cbtype.toString().toLowerCase()+"]";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFGraph.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFGraph.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFGraph.java
deleted file mode 100644
index 99b177d..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFGraph.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-
-public class GDFGraph
-{
-
-
- private ArrayList<GDFNode> _roots = null;
- private Program _rtprog = null;
-
-
- public GDFGraph( Program prog, ArrayList<GDFNode> roots )
- {
- _rtprog = prog;
- _roots = roots;
- }
-
- public ArrayList<GDFNode> getGraphRootNodes()
- {
- return _roots;
- }
-
- public Program getRuntimeProgram()
- {
- return _rtprog;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFLoopNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFLoopNode.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFLoopNode.java
deleted file mode 100644
index 55c3ed1..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFLoopNode.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-
-public class GDFLoopNode extends GDFNode
-{
-
- private GDFNode _predicate = null;
- private HashMap<String,GDFNode> _linputs = null; //all read variables
- private HashMap<String,GDFNode> _loutputs = null; //all updated variables, not necessarily liveout
-
- public GDFLoopNode( ProgramBlock pb, GDFNode predicate, HashMap<String, GDFNode> inputs, HashMap<String,GDFNode> outputs )
- {
- super(null, pb, new ArrayList<GDFNode>(inputs.values()));
- _type = NodeType.LOOP_NODE;
- _predicate = predicate;
- _linputs = inputs;
- _loutputs = outputs;
- }
-
- public HashMap<String, GDFNode> getLoopInputs()
- {
- return _linputs;
- }
-
- public HashMap<String, GDFNode> getLoopOutputs()
- {
- return _loutputs;
- }
-
- public GDFNode getLoopPredicate()
- {
- return _predicate;
- }
-
- @Override
- public String explain(String deps)
- {
- String ldeps = (deps!=null) ? deps : "";
-
- //current node details
- return "LoopNode "+ldeps+" ["+_linputs.keySet().toString()+","+_loutputs.keySet().toString()+"]";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFNode.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFNode.java
deleted file mode 100644
index 8a048cd..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GDFNode.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-
-/**
- * The reason of a custom graph structure is to unify both within DAG
- * and cross DAG enumeration. Conceptually, we would only need interesting
- * properties of transient reads and could compile locally.
- *
- * Furthermore, having a global graph structure also allows for more advanced
- * algebraic simplification rewrites because the semantics of transient read
- * inputs are always available.
- *
- */
-public class GDFNode
-{
-
- public enum NodeType{
- HOP_NODE,
- LOOP_NODE,
- CROSS_BLOCK_NODE,
- }
-
- private static IDSequence _seqID = new IDSequence();
-
- protected NodeType _type = null;
- protected long _ID = -1;
-
- //references to original program and hop dag
- protected Hop _hop = null;
- protected ProgramBlock _pb = null;
-
- //input nodes
- protected ArrayList<GDFNode> _inputs = null;
-
- public GDFNode()
- {
- _ID = _seqID.getNextID();
- }
-
- public GDFNode( Hop hop, ProgramBlock pb, ArrayList<GDFNode> inputs )
- {
- this();
- _type = NodeType.HOP_NODE;
- _hop = hop;
- _pb = pb;
- _inputs = inputs;
- }
-
- public NodeType getNodeType()
- {
- return _type;
- }
-
- public long getID()
- {
- return _ID;
- }
-
- public Hop getHop()
- {
- return _hop;
- }
-
- public ProgramBlock getProgramBlock()
- {
- return _pb;
- }
-
- public Program getProgram()
- {
- if( _pb != null )
- return _pb.getProgram();
- return null;
- }
-
- public ArrayList<GDFNode> getInputs()
- {
- return _inputs;
- }
-
- public DataType getDataType()
- {
- return _hop.getDataType();
- }
-
- /**
- * If the output or any input is a matrix we need to consider
- * MR configurations. This for examples excludes Literals or
- * any purely scalar operation.
- *
- * @return
- */
- public boolean requiresMREnumeration()
- {
- //general rule: MR generation required if at least one matrix input/output
- boolean ret = (_hop.getDataType() == DataType.MATRIX);
- for( Hop c : _hop.getInput() )
- ret |= (c.getDataType() == DataType.MATRIX);
-
- //special cases of CP-only operators
- if( _hop instanceof UnaryOp && ((UnaryOp)_hop).getOp()==OpOp1.CAST_AS_SCALAR ) //as.scalar
- ret = false;
- if( _hop instanceof DataGenOp && ((DataGenOp)_hop).getOp()==DataGenMethod.SINIT ) //matrix(str, )
- ret = false;
- if( _hop instanceof UnaryOp && ((UnaryOp)_hop).getOp()==OpOp1.NROW ) //nrow - meta data only
- ret = false;
- if( _hop instanceof UnaryOp && ((UnaryOp)_hop).getOp()==OpOp1.NCOL ) //ncol - meta data only
- ret = false;
-
- return ret;
- }
-
- /**
- *
- * @param format
- * @return
- */
- public boolean isValidInputFormatForOperation( FileFormatTypes format )
- {
- return ( _hop instanceof UnaryOp && format!=FileFormatTypes.CSV
- || (_hop instanceof AggUnaryOp && ((AggUnaryOp)_hop).getDirection()==Direction.RowCol && format!=FileFormatTypes.CSV)
- || (_hop instanceof ReorgOp && ((ReorgOp)_hop).getOp()==ReOrgOp.TRANSPOSE && format!=FileFormatTypes.CSV)
- || format==FileFormatTypes.BINARY ); //any op
- }
-
- /**
- *
- * @param deps
- * @return
- */
- public String explain(String deps)
- {
- String ldeps = (deps!=null) ? deps : "";
-
- //node details
- if( _hop!=null )
- return "Node "+ldeps+" ["+_hop.getHopID()+", "+_hop.getOpString()+"]";
- else
- return "Node "+ldeps+" [null]";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GraphBuilder.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GraphBuilder.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GraphBuilder.java
deleted file mode 100644
index 6507faa..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfgraph/GraphBuilder.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.globalopt.Summary;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.utils.Explain;
-
-/**
- * GENERAL 'GDF GRAPH' STRUCTURE, by MB:
- * 1) Each hop is represented by an GDFNode
- * 2) Each loop is represented by a structured GDFLoopNode
- * 3) Transient Read/Write connections are represented via CrossBlockNodes,
- * a) type PLAIN: single input crossblocknode represents unconditional data flow
- * b) type MERGE: two inputs crossblocknode represent conditional data flow merge
- *
- * In detail, the graph builder essentially does a single pass over the entire program
- * and constructs the global data flow graph bottom up. We create crossblocknodes for
- * every transient write, loop nodes for for/while programblocks, and crossblocknodes
- * after every if programblock.
- *
- */
-public class GraphBuilder
-{
-
- private static final boolean IGNORE_UNBOUND_UPDATED_VARS = true;
-
- /**
- *
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- */
- public static GDFGraph constructGlobalDataFlowGraph( Program prog, Summary summary )
- throws DMLRuntimeException, HopsException
- {
- Timing time = new Timing(true);
-
- HashMap<String, GDFNode> roots = new HashMap<String, GDFNode>();
- for( ProgramBlock pb : prog.getProgramBlocks() )
- constructGDFGraph( pb, roots );
-
- //create GDF graph root nodes
- ArrayList<GDFNode> ret = new ArrayList<GDFNode>();
- for( GDFNode root : roots.values() )
- if( !(root instanceof GDFCrossBlockNode) )
- ret.add(root);
-
- //create GDF graph
- GDFGraph graph = new GDFGraph(prog, ret);
-
- summary.setTimeGDFGraph(time.stop());
- return graph;
- }
-
- /**
- *
- * @param pb
- * @param roots
- * @throws DMLRuntimeException
- * @throws HopsException
- */
- @SuppressWarnings("unchecked")
- private static void constructGDFGraph( ProgramBlock pb, HashMap<String, GDFNode> roots )
- throws DMLRuntimeException, HopsException
- {
- if (pb instanceof FunctionProgramBlock )
- {
- throw new DMLRuntimeException("FunctionProgramBlocks not implemented yet.");
- }
- else if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- WhileStatementBlock wsb = (WhileStatementBlock) pb.getStatementBlock();
- //construct predicate node (conceptually sequence of from/to/incr)
- GDFNode pred = constructGDFGraph(wsb.getPredicateHops(), wpb, new HashMap<Long, GDFNode>(), roots);
- HashMap<String,GDFNode> inputs = constructLoopInputNodes(wpb, wsb, roots);
- HashMap<String,GDFNode> lroots = (HashMap<String, GDFNode>) inputs.clone();
- //process childs blocks
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- constructGDFGraph(pbc, lroots);
- HashMap<String,GDFNode> outputs = constructLoopOutputNodes(wsb, lroots);
- GDFLoopNode lnode = new GDFLoopNode(wpb, pred, inputs, outputs );
- //construct crossblock nodes
- constructLoopOutputCrossBlockNodes(wsb, lnode, outputs, roots, wpb);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- IfStatementBlock isb = (IfStatementBlock) pb.getStatementBlock();
- //construct predicate
- if( isb.getPredicateHops()!=null ) {
- Hop pred = isb.getPredicateHops();
- roots.put(pred.getName(), constructGDFGraph(pred, ipb, new HashMap<Long,GDFNode>(), roots));
- }
- //construct if and else branch separately
- HashMap<String,GDFNode> ifRoots = (HashMap<String, GDFNode>) roots.clone();
- HashMap<String,GDFNode> elseRoots = (HashMap<String, GDFNode>) roots.clone();
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- constructGDFGraph(pbc, ifRoots);
- if( ipb.getChildBlocksElseBody()!=null )
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- constructGDFGraph(pbc, elseRoots);
- //merge data flow roots (if no else, elseRoots refer to original roots)
- reconcileMergeIfProgramBlockOutputs(ifRoots, elseRoots, roots, ipb);
- }
- else if (pb instanceof ForProgramBlock) //incl parfor
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- ForStatementBlock fsb = (ForStatementBlock)pb.getStatementBlock();
- //construct predicate node (conceptually sequence of from/to/incr)
- GDFNode pred = constructForPredicateNode(fpb, fsb, roots);
- HashMap<String,GDFNode> inputs = constructLoopInputNodes(fpb, fsb, roots);
- HashMap<String,GDFNode> lroots = (HashMap<String, GDFNode>) inputs.clone();
- //process childs blocks
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- constructGDFGraph(pbc, lroots);
- HashMap<String,GDFNode> outputs = constructLoopOutputNodes(fsb, lroots);
- GDFLoopNode lnode = new GDFLoopNode(fpb, pred, inputs, outputs );
- //construct crossblock nodes
- constructLoopOutputCrossBlockNodes(fsb, lnode, outputs, roots, fpb);
- }
- else //last-level program block
- {
- StatementBlock sb = pb.getStatementBlock();
- ArrayList<Hop> hops = sb.get_hops();
- if( hops != null )
- {
- //create new local memo structure for local dag
- HashMap<Long, GDFNode> lmemo = new HashMap<Long, GDFNode>();
- for( Hop hop : hops )
- {
- //recursively construct GDF graph for hop dag root
- GDFNode root = constructGDFGraph(hop, pb, lmemo, roots);
- if( root == null )
- throw new HopsException( "GDFGraphBuilder: failed to constuct dag root for: "+Explain.explain(hop) );
-
- //create cross block nodes for all transient writes
- if( hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTWRITE )
- root = new GDFCrossBlockNode(hop, pb, root, hop.getName());
-
- //add GDF root node to global roots
- roots.put(hop.getName(), root);
- }
- }
-
- }
- }
-
- /**
- *
- * @param hop
- * @param pb
- * @param lmemo
- * @param roots
- * @return
- */
- private static GDFNode constructGDFGraph( Hop hop, ProgramBlock pb, HashMap<Long, GDFNode> lmemo, HashMap<String, GDFNode> roots )
- {
- if( lmemo.containsKey(hop.getHopID()) )
- return lmemo.get(hop.getHopID());
-
- //process childs recursively first
- ArrayList<GDFNode> inputs = new ArrayList<GDFNode>();
- for( Hop c : hop.getInput() )
- inputs.add( constructGDFGraph(c, pb, lmemo, roots) );
-
- //connect transient reads to existing roots of data flow graph
- if( hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTREAD ){
- inputs.add(roots.get(hop.getName()));
- }
-
- //add current hop
- GDFNode gnode = new GDFNode(hop, pb, inputs);
-
- //add GDF node of updated variables to global roots (necessary for loops, where updated local
- //variables might never be bound to their logical variables names
- if( !IGNORE_UNBOUND_UPDATED_VARS ) {
- //NOTE: currently disabled because unnecessary, if no transientwrite by definition included in other transientwrite
- if( pb.getStatementBlock()!=null && pb.getStatementBlock().variablesUpdated().containsVariable(hop.getName()) ) {
- roots.put(hop.getName(), gnode);
- }
- }
-
- //memoize current node
- lmemo.put(hop.getHopID(), gnode);
-
- return gnode;
- }
-
- /**
- *
- * @param fpb
- * @param fsb
- * @param roots
- * @return
- */
- private static GDFNode constructForPredicateNode(ForProgramBlock fpb, ForStatementBlock fsb, HashMap<String, GDFNode> roots)
- {
- HashMap<Long, GDFNode> memo = new HashMap<Long, GDFNode>();
- GDFNode from = (fsb.getFromHops()!=null)? constructGDFGraph(fsb.getFromHops(), fpb, memo, roots) : null;
- GDFNode to = (fsb.getToHops()!=null)? constructGDFGraph(fsb.getToHops(), fpb, memo, roots) : null;
- GDFNode incr = (fsb.getIncrementHops()!=null)? constructGDFGraph(fsb.getIncrementHops(), fpb, memo, roots) : null;
- ArrayList<GDFNode> inputs = new ArrayList<GDFNode>();
- inputs.add(from);
- inputs.add(to);
- inputs.add(incr);
- //TODO for predicates
- GDFNode pred = new GDFNode(null, fpb, inputs );
-
- return pred;
- }
-
- /**
- *
- * @param fpb
- * @param fsb
- * @param roots
- * @return
- * @throws DMLRuntimeException
- */
- private static HashMap<String, GDFNode> constructLoopInputNodes( ProgramBlock fpb, StatementBlock fsb, HashMap<String, GDFNode> roots )
- throws DMLRuntimeException
- {
- HashMap<String, GDFNode> ret = new HashMap<String, GDFNode>();
- Set<String> invars = fsb.variablesRead().getVariableNames();
- for( String var : invars ) {
- if( fsb.liveIn().containsVariable(var) ) {
- GDFNode node = roots.get(var);
- if( node == null )
- throw new DMLRuntimeException("GDFGraphBuilder: Non-existing input node for variable: "+var);
- ret.put(var, node);
- }
- }
-
- return ret;
- }
-
- private static HashMap<String, GDFNode> constructLoopOutputNodes( StatementBlock fsb, HashMap<String, GDFNode> roots )
- throws HopsException
- {
- HashMap<String, GDFNode> ret = new HashMap<String, GDFNode>();
-
- Set<String> outvars = fsb.variablesUpdated().getVariableNames();
- for( String var : outvars )
- {
- GDFNode node = roots.get(var);
-
- //handle non-existing nodes
- if( node == null ) {
- if( !IGNORE_UNBOUND_UPDATED_VARS )
- throw new HopsException( "GDFGraphBuilder: failed to constuct loop output for variable: "+var );
- else
- continue; //skip unbound updated variables
- }
-
- //add existing node to loop outputs
- ret.put(var, node);
- }
-
- return ret;
- }
-
- /**
- *
- * @param ifRoots
- * @param elseRoots
- * @param roots
- * @param pb
- */
- private static void reconcileMergeIfProgramBlockOutputs( HashMap<String, GDFNode> ifRoots, HashMap<String, GDFNode> elseRoots, HashMap<String, GDFNode> roots, IfProgramBlock pb )
- {
- //merge same variable names, different data
- //( incl add new vars from if branch if node2==null)
- for( Entry<String, GDFNode> e : ifRoots.entrySet() ){
- GDFNode node1 = e.getValue();
- GDFNode node2 = elseRoots.get(e.getKey()); //original or new
- if( node1 != node2 )
- node1 = new GDFCrossBlockNode(null, pb, node1, node2, e.getKey() );
- roots.put(e.getKey(), node1);
- }
-
- //add new vars from else branch
- for( Entry<String, GDFNode> e : elseRoots.entrySet() ){
- if( !ifRoots.containsKey(e.getKey()) )
- roots.put(e.getKey(), e.getValue());
- }
- }
-
- /**
- *
- * @param sb
- * @param loop
- * @param loutputs
- * @param roots
- * @param pb
- */
- private static void constructLoopOutputCrossBlockNodes(StatementBlock sb, GDFLoopNode loop, HashMap<String, GDFNode> loutputs, HashMap<String, GDFNode> roots, ProgramBlock pb)
- {
- //iterate over all output (updated) variables
- for( Entry<String,GDFNode> e : loutputs.entrySet() )
- {
- //create crossblocknode, if updated variable is also in liveout
- if( sb.liveOut().containsVariable(e.getKey()) ) {
- GDFCrossBlockNode node = null;
- if( roots.containsKey(e.getKey()) )
- node = new GDFCrossBlockNode(null, pb, roots.get(e.getKey()), loop, e.getKey()); //MERGE
- else
- node = new GDFCrossBlockNode(null, pb, loop, e.getKey()); //PLAIN
- roots.put(e.getKey(), node);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
deleted file mode 100644
index 10f4df1..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
-
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
-
-public abstract class GDFMismatchHeuristic
-{
-
- public enum MismatchHeuristicType {
- FIRST,
- BLOCKSIZE_OR_FIRST,
- }
-
- /**
- * Returns the name of the implementing mismatch heuristic.
- *
- * @return
- */
- public abstract String getName();
-
- /**
- * Resolve the mismatch of two given rewrite configurations. This call returns true,
- * if and only if the new configuration is chosen.
- *
- * @param currRc
- * @param newRc
- * @return
- */
- public abstract boolean resolveMismatch( RewriteConfig currRc, RewriteConfig newRc );
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
deleted file mode 100644
index b9604f4..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
-
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
-
-public class GDFMismatchHeuristicBlocksizeOrFirst extends GDFMismatchHeuristic
-{
-
- @Override
- public String getName(){
- return "BLOCKSIZE_OR_FIRST";
- }
-
- @Override
- public boolean resolveMismatch( RewriteConfig currRc, RewriteConfig newRc )
- {
- //check for blocksize mismatch
- if( currRc.getBlockSize() != newRc.getBlockSize() )
- {
- //choose the new rewrite config if its blocksize is larger than
- //the current (intuition: we generally prefer larger blocksizes
- //because this often enables better physical operators with constraints
- //like ncol(X) <= blocksize)
- return (currRc.getBlockSize() < newRc.getBlockSize());
- }
-
- //return the current rewrite configuration (first come first served)
- //if the previous check for blocksize mismatch failed
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
deleted file mode 100644
index d069aa8..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
-
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
-
-public class GDFMismatchHeuristicFirst extends GDFMismatchHeuristic
-{
-
- @Override
- public String getName(){
- return "FIRST";
- }
-
- @Override
- public boolean resolveMismatch( RewriteConfig currRc, RewriteConfig newRc )
- {
- //always return the current rewrite configuration (first come first served)
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java b/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
deleted file mode 100644
index 0031a87..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
-
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-public abstract class MismatchHeuristicFactory
-{
-
- /**
- *
- * @param type
- * @return
- * @throws DMLRuntimeException
- */
- public static GDFMismatchHeuristic createMismatchHeuristic( MismatchHeuristicType type )
- throws DMLRuntimeException
- {
- switch( type ) {
- case FIRST:
- return new GDFMismatchHeuristicFirst();
- case BLOCKSIZE_OR_FIRST:
- return new GDFMismatchHeuristicBlocksizeOrFirst();
-
- default:
- throw new DMLRuntimeException("Unsupported mismatch heuristic: "+type);
- }
- }
-}
[58/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/LengthTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/LengthTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/LengthTest.java
index a2885c5..57c34cf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/LengthTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/LengthTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MaxTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MaxTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MaxTest.java
index 24e86c8..0491782 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MaxTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MaxTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MinTest.java
index afce265..91d308f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/MinTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NColTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NColTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NColTest.java
index 26b11bb..0f76f1c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NColTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NColTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NRowTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NRowTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NRowTest.java
index a500242..5086671 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NRowTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/NRowTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ProdTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ProdTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ProdTest.java
index e8169ad..8f3b596 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ProdTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ProdTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumTest.java
index 7c215d1..59a8c6d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumsSqTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumsSqTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumsSqTest.java
index 9e8b4d3..559e853 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumsSqTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowSumsSqTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumSqTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumSqTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumSqTest.java
index a1867ab..393c010 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumSqTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumSqTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumTest.java
index 712505f..8427960 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/SumTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/TraceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/TraceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/TraceTest.java
index 8f87d53..ad1441d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/TraceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/TraceTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendChainTest.java
index c128b8a..875d5f2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendChainTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class AppendChainTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
index b65fdd1..6c2f245 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import java.util.HashMap;
import java.util.Random;
@@ -23,16 +23,16 @@ import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.BinaryOp.AppendMethod;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.BinaryOp.AppendMethod;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class AppendMatrixTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
index 0610d10..b4478d4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import java.util.HashMap;
import java.util.Random;
@@ -23,14 +23,14 @@ import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class AppendVectorTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/append/RBindCBindMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/RBindCBindMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/RBindCBindMatrixTest.java
index 1464bcf..1d49994 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/RBindCBindMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/RBindCBindMatrixTest.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class RBindCBindMatrixTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/append/StringAppendTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/StringAppendTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/StringAppendTest.java
index d4f3f6f..2b5e9a9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/StringAppendTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/StringAppendTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class StringAppendTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/BinUaggChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/BinUaggChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/BinUaggChainTest.java
index 83b676e..995b4f4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/BinUaggChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/BinUaggChainTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* TODO: extend test by various binary operator - unary aggregate operator combinations.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CentralMomentTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CentralMomentTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CentralMomentTest.java
index f2f2fe4..38a69b0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CentralMomentTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CentralMomentTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CovarianceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CovarianceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CovarianceTest.java
index b567066..e906989 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CovarianceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/CovarianceTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/DiagMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/DiagMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/DiagMatrixMultiplicationTest.java
index 05795cf..136085b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/DiagMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/DiagMatrixMultiplicationTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class DiagMatrixMultiplicationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
index 6990da8..184f188 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.Date;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ElementwiseAdditionMultiplicationTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionTest.java
index d044f1e..112dfdf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseDivisionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseDivisionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseDivisionTest.java
index 2175aaf..4f4d852 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseDivisionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseDivisionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseModulusTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseModulusTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseModulusTest.java
index 062d789..ad8a5aa 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseModulusTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseModulusTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.functionobjects.Modulus;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.functionobjects.Modulus;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ElementwiseModulusTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseMultiplicationTest.java
index 612af46..88af59a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseMultiplicationTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ElementwiseMultiplicationTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseSubtractionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseSubtractionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseSubtractionTest.java
index 7fd2748..21b54ae 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseSubtractionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseSubtractionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ElementwiseSubtractionTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
index 2d82425..a393733 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class MapMultChainTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultLimitTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultLimitTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultLimitTest.java
index 5040ca2..aba157e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultLimitTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultLimitTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
/**
* Tests the number of mapmult operations that can be piggybacked into the same GMR job.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixMultiplicationTest.java
index 2dc75f6..dc797c4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixMultiplicationTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class MatrixMultiplicationTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixVectorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixVectorTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixVectorTest.java
index 47e1dd1..4ad254a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixVectorTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MatrixVectorTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class MatrixVectorTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/OuterProductTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/OuterProductTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/OuterProductTest.java
index 1142ca7..667f4ee 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/OuterProductTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/OuterProductTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class OuterProductTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/QuantileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/QuantileTest.java
index 59098d7..7fa00e9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/QuantileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/QuantileTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarAdditionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarAdditionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarAdditionTest.java
index a7301b8..a76b332 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarAdditionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarAdditionTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarDivisionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarDivisionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarDivisionTest.java
index 98c286b..43997f4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarDivisionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarDivisionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarModulusTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarModulusTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarModulusTest.java
index e8d4d76..76462ec 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarModulusTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarModulusTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.functionobjects.Modulus;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.runtime.functionobjects.Modulus;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ScalarModulusTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarMultiplicationTest.java
index 7db6f61..6d68c05 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarMultiplicationTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ScalarMultiplicationTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarSubtractionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarSubtractionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarSubtractionTest.java
index 5ba6218..94938b8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarSubtractionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ScalarSubtractionTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ScalarSubtractionTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/TransposeMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/TransposeMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/TransposeMatrixMultiplicationTest.java
index 6864f76..6c2d89d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/TransposeMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/TransposeMatrixMultiplicationTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* This test investigates the specific Hop-Lop rewrite t(X)%*%v -> t(t(v)%*%X).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
index 5bf219f..efc69a2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.UAggOuterChain;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
* TODO: extend test by various binary operator - unary aggregate operator combinations.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UltraSparseMRMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UltraSparseMRMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UltraSparseMRMatrixMultiplicationTest.java
index 2ec6cf2..5feba2a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UltraSparseMRMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UltraSparseMRMatrixMultiplicationTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggBinaryOp.MMultMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggBinaryOp.MMultMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Test for MMCJ MR because otherwise seldom (if at all) executed in our testsuite, ultrasparse
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ZipMMSparkMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ZipMMSparkMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ZipMMSparkMatrixMultiplicationTest.java
index 4bd6540..5cc4646 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ZipMMSparkMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ZipMMSparkMatrixMultiplicationTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggBinaryOp.MMultMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggBinaryOp.MMultMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
index 3a8b172..5de1c82 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixMatrixCellwiseOperationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperationTest.java
index a693a0f..8adc9a7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperationTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixVectorColCellwiseOperationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperationTest.java
index bca24e2..6c7eeeb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperationTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixVectorRowCellwiseOperationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMinus1MultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMinus1MultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMinus1MultTest.java
index c27ba69..80ea5d8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMinus1MultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullMinus1MultTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseCompareOperationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseCompareOperationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseCompareOperationTest.java
index a693d5d..fc51c47 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseCompareOperationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseCompareOperationTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.Arrays;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* TODO cleanup outer(X,Y,z) definition to take two column vectors instead of column and row vector.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseOperationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseOperationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseOperationTest.java
index 3fdf5be..9d5b6a6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseOperationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/FullVectorVectorCellwiseOperationTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* TODO cleanup outer(X,Y,z) definition to take two column vectors instead of column and row vector.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
index 5198b9e..590a69e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,15 +23,15 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggBinaryOp.MMultMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggBinaryOp.MMultMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullDistributedMatrixMultiplicationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
index 42f7cee..93f0228 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.io.IOException;
import java.util.HashMap;
@@ -24,17 +24,17 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullIntegerDivisionTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
index 67182af..49da3d2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,12 +23,12 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixMultiplicationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
index b3c1496..92d2ed9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,13 +23,13 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixMultiplicationTransposeSelfTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
index 30f4534..81d4919 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,12 +23,12 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMatrixMultiplicationUltraSparseTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
index 408fba7..36acec6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.io.IOException;
import java.util.HashMap;
@@ -24,17 +24,17 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullMinMaxComparisonTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredMatrixTest.java
index c9166cd..0aa822f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredMatrixTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,13 +23,13 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* The main purpose of this test is to verify various input combinations for
[23/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java b/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
deleted file mode 100644
index f8ce6f1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.compile;
-
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-
-/**
- * This enumeration defines the set of all map-reduce job types. Each job type
- * is associated with following properties:
- *
- * id - Unique identifier.
- *
- * name - Job name.
- *
- * producesIntermediateOutput - set to false if the the job produces an
- * intermediate output that MUST be consumed by subsequent jobs. The
- * intermediate output is NEVER seen by the end user.
- *
- * emptyInputsAllowed - defines whether or not the job can take an empty input
- * file as an input. Currently, this flag is set to true only for RAND jobs.
- *
- * allowsSingleShuffleInstruction - set to true if the job allows only a single
- * instruction in the shuffle phase. For example, jobs that perform matrix
- * multiplication (MMCJ,MMRJ) can perform only one multiplication per job.
- * Allowing multiple multiplications within a single job complicates
- * the implementation (due to specialized key-value pairs for each multiplication)
- * and such a combination can potentially hinder the performance (since these jobs
- * make use of a lot of memory). Similarly, SORT job can sort a single stream of values.
- *
- */
-
-public enum JobType
-{
- /* Add new job types to the following list */
- // (id, name, emptyInputsAllowed, allowsSingleShuffleInstruction, allowsNoOtherInstructions)
- INVALID (-1, "INVALID", false, false, false),
- ANY (0, "ANY", false, false, false),
- GMR (1, "GMR", false, false, false),
- DATAGEN (2, "DATAGEN", true, false, false),
- REBLOCK (3, "REBLOCK", false, false, false),
- MMCJ (4, "MMCJ", false, true, false),
- MMRJ (5, "MMRJ", false, false, false),
- COMBINE (6, "COMBINE", false, false, true),
- SORT (7, "SORT", false, true, true), // allows only "InstructionsBeforeSort" and nothing else.
- CM_COV (8, "CM_COV", false, false, false), // allows only instructions in the mapper
- GROUPED_AGG (9, "GROUPED_AGG", false, false, false),
- //PARTITION (10, "PARTITION", false, false, true), // MB: meta learning removed
- DATA_PARTITION (11, "DATAPARTITION", false, false, true),
- CSV_REBLOCK (12, "CSV_REBLOCK", false, false, false),
- CSV_WRITE (13, "CSV_WRITE", false, false, true),
- TRANSFORM (14, "TRANSFORM", false, true, false),
- GMRCELL (15, "GMRCELL", false, false, false);
-
-
-
- private static int maxJobID = -1;
- static {
- for(JobType jt : JobType.values())
- {
- if(jt.getId() > maxJobID)
- maxJobID = jt.getId();
- }
- }
- /* Following code should not be edited when adding a new job type */
-
- private final int id;
- private final String name;
-
- private final boolean emptyInputsAllowed;
-
- private final boolean allowsSingleShuffleInstruction;
-
- /**
- * Indicates whether a job can piggyback "other" operations.
- * For example, COMBINE job can only piggyback multiple combine operators but can not perform any other operations.
- */
- private final boolean allowsNoOtherInstructions;
-
- JobType(int id, String name, boolean aei, boolean assi, boolean anoi) {
- this.id = id;
- this.name = name;
- this.emptyInputsAllowed = aei;
- this.allowsSingleShuffleInstruction = assi;
- this.allowsNoOtherInstructions = anoi;
- }
-
- public int getId() {
- return id;
- }
-
- public String getName() {
- return name;
- }
-
-/* public boolean producesIntermediateOutput() {
- return producesIntermediateOutput;
- }
-*/
- public boolean areEmptyInputsAllowed() {
- return emptyInputsAllowed;
- }
-
- public boolean allowsSingleShuffleInstruction() {
- return allowsSingleShuffleInstruction;
- }
-
- public boolean allowsNoOtherInstructions() {
- return allowsNoOtherInstructions;
- }
-
- public Lop.Type getShuffleLopType() throws DMLRuntimeException {
- if ( !allowsSingleShuffleInstruction )
- throw new DMLRuntimeException("Shuffle Lop Type is not defined for a job (" + getName() + ") with allowsSingleShuffleInstruction=false.");
- else {
- if ( getName().equals("MMCJ") )
- return Lop.Type.MMCJ;
- else if ( getName().equals("MMRJ") )
- return Lop.Type.MMRJ;
- else if ( getName().equals("SORT") )
- return Lop.Type.SortKeys;
- else if ( getName().equals("TRANSFORM"))
- return Lop.Type.ParameterizedBuiltin;
- else
- throw new DMLRuntimeException("Shuffle Lop Type is not defined for a job (" + getName() + ") that allows a single shuffle instruction.");
- }
- }
-
- public static JobType findJobTypeFromLop(Lop node) {
- Lop.Type lt = node.getType();
- switch(lt) {
- case DataGen: return JobType.DATAGEN;
-
- case ReBlock: return JobType.REBLOCK;
-
- case Grouping: return JobType.GMR;
-
- case MMCJ: return JobType.MMCJ;
-
- case MMRJ: return JobType.MMRJ;
-
- case MMTSJ: return JobType.GMR;
-
- case SortKeys: return JobType.SORT;
-
- case CentralMoment:
- case CoVariance:
- return JobType.CM_COV;
-
- case GroupedAgg: return JobType.GROUPED_AGG;
-
- case CombineBinary:
- case CombineTernary:
- return JobType.COMBINE;
-
- case DataPartition: return JobType.DATA_PARTITION;
-
- case CSVReBlock: return JobType.CSV_REBLOCK;
-
- case ParameterizedBuiltin:
- if( ((ParameterizedBuiltin)node).getOp() == ParameterizedBuiltin.OperationTypes.TRANSFORM )
- return JobType.TRANSFORM;
-
- case Data:
- /*
- * Only Write LOPs with external data formats (except MatrixMarket) produce MR Jobs
- */
- FileFormatTypes fmt = ((Data) node).getFileFormatType();
- if ( fmt == FileFormatTypes.CSV )
- return JobType.CSV_WRITE;
- else
- return null;
-
- default:
- return null;
- }
- }
-
- public boolean isCompatibleWithParentNodes()
- throws DMLRuntimeException
- {
- if ( !allowsSingleShuffleInstruction )
- throw new DMLRuntimeException("isCompatibleWithParentNodes() can not be invoked for a job (" + getName() + ") with allowsSingleShuffleInstruction=false.");
- else {
- if ( getName().equals("MMCJ") )
- return false;
- else if ( getName().equals("MMRJ") || getName().equals("SORT") || getName().equals("TRANSFORM"))
- return true;
- else
- throw new DMLRuntimeException("Implementation for isCompatibleWithParentNodes() is missing for a job (" + getName() + ") that allows a single shuffle instruction.");
- }
- }
-
- public boolean allowsRecordReaderInstructions() {
- if ( getName().equals("GMR") )
- return true;
- else
- return false;
- }
-
- public int getBase() {
- if (id == -1)
- return 0;
- else if (id == 0) {
- // for ANY, return the bit vector with x number of 1's,
- // where x = number of actual job types (i.e., excluding INVALID,ANY)
- //System.out.println("ANY --> " + JobType.values().length + ", " + (Math.pow(2, JobType.values().length-2)-1) + ", " + (Math.pow(2,13-2)-1));
- return (int) Math.pow(2, maxJobID)-1;
- }
- else
- return (int) Math.pow(2, id-1);
- }
-
- public JobType findJobTypeById(int id) {
- for (JobType jt : JobType.values()) {
- if (jt.getId() == id)
- return jt;
- }
- return null;
- }
-
- public JobType findJobTypeByName(String name) {
- for (JobType jt : JobType.values()) {
- if (jt.getName().equalsIgnoreCase(name))
- return jt;
- }
- return null;
- }
-
- public static int getNumJobTypes() {
- return values().length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java b/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
deleted file mode 100644
index c5099af..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.compile;
-
-import java.util.Comparator;
-
-import com.ibm.bi.dml.lops.Lop;
-
-
-/**
- *
- * Comparator class used in sorting the LopDAG in topological order. Refer to
- * doTopologicalSort_strict_order() in dml/lops/compile/Dag.java
- *
- * Topological sort guarantees the following:
- *
- * 1) All lops with level i appear before any lop with level greater than i
- * (source nodes are at level 0)
- *
- * 2) Within a given level, nodes are ordered by their ID i.e., by the other in
- * which they are created
- *
- * compare() method is designed to respect the above two requirements.
- *
- * @param <N>
- */
-public class LopComparator<N extends Lop>
- implements Comparator<N>
-{
-
- @Override
- public int compare(N o1, N o2) {
- if (o1.getLevel() < o2.getLevel())
- return -1; // o1 is less than o2
- else if (o1.getLevel() > o2.getLevel())
- return 1; // o1 is greater than o2
- else {
- if (o1.getID() < o2.getID())
- return -1; // o1 is less than o2
- else if (o1.getID() > o2.getID())
- return 1; // o1 is greater than o2
- else
- throw new RuntimeException("Unexpected error: ID's of two lops are same.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java b/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
deleted file mode 100644
index 804fe2f..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
+++ /dev/null
@@ -1,591 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.runtime;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.wink.json4j.JSONException;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.io.MatrixWriterFactory;
-import com.ibm.bi.dml.runtime.matrix.CMCOVMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CombineMR;
-import com.ibm.bi.dml.runtime.matrix.DataGenMR;
-import com.ibm.bi.dml.runtime.matrix.DataPartitionMR;
-import com.ibm.bi.dml.runtime.matrix.GMR;
-import com.ibm.bi.dml.runtime.matrix.GroupedAggMR;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MMCJMR;
-import com.ibm.bi.dml.runtime.matrix.MMRJMR;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.ReblockMR;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.WriteCSVMR;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
-import com.ibm.bi.dml.runtime.transform.DataTransform;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-
-
-public class RunMRJobs
-{
-
- /**
- * Wrapper for submitting MR job instructions incl preparation and actual submission.
- * The preparation includes (1) pulling stats out of symbol table and populating the
- * instruction, (2) instruction patching, and (3) export of in-memory matrices if
- * required.
- *
- * Furthermore, this wrapper also provides a hook for runtime piggybacking to intercept
- * concurrent job submissions in order to collect and merge instructions.
- *
- * @param inst
- * @param ec
- * @return
- */
-
- public static JobReturn prepareAndSubmitJob( MRJobInstruction inst, ExecutionContext ec )
- throws DMLRuntimeException
- {
- // Obtain references to all input matrices
- MatrixObject[] inputMatrices = inst.extractInputMatrices(ec);
-
- // export dirty matrices to HDFS
- // note: for REBLOCK postponed until we know if necessary
- if( !(inst.getJobType() == JobType.REBLOCK) )
- {
- //export matrices
- for(MatrixObject m : inputMatrices) {
- if ( m.isDirty() || m.getRDDHandle()!=null )
- m.exportData();
- }
-
- //check input files
- checkEmptyInputs( inst, inputMatrices );
- }
-
- // Obtain references to all output matrices
- inst.extractOutputMatrices(ec);
-
- // obtain original state
- String rdInst = inst.getIv_randInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String shuffleInst = inst.getIv_shuffleInstructions();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
-
- // variable patching (replace placeholders with variables)
- inst.setIv_randInstructions(updateLabels(rdInst, ec.getVariables()));
- inst.setIv_recordReaderInstructions(updateLabels(rrInst, ec.getVariables()));
- inst.setIv_instructionsInMapper(updateLabels(mapInst, ec.getVariables()));
- inst.setIv_shuffleInstructions(updateLabels(shuffleInst, ec.getVariables()));
- inst.setIv_aggInstructions(updateLabels(aggInst, ec.getVariables()));
- inst.setIv_otherInstructions(updateLabels(otherInst, ec.getVariables()));
-
- // runtime piggybacking if applicable
- JobReturn ret = null;
- if( OptimizerUtils.ALLOW_RUNTIME_PIGGYBACKING
- && RuntimePiggybacking.isActive()
- && RuntimePiggybacking.isSupportedJobType(inst.getJobType()) )
- {
- ret = RuntimePiggybacking.submitJob( inst );
- }
- else
- ret = submitJob( inst );
-
- // reset original state
- inst.setIv_randInstructions(rdInst);
- inst.setIv_recordReaderInstructions(rrInst);
- inst.setIv_instructionsInMapper(mapInst);
- inst.setIv_shuffleInstructions(shuffleInst);
- inst.setIv_aggInstructions(aggInst);
- inst.setIv_otherInstructions(otherInst);
-
- return ret;
- }
-
- /**
- * Submits an MR job instruction, without modifying any state of that instruction.
- *
- * @param inst
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- public static JobReturn submitJob(MRJobInstruction inst )
- throws DMLRuntimeException
- {
- JobReturn ret = new JobReturn();
- MatrixObject[] inputMatrices = inst.getInputMatrices();
- MatrixObject[] outputMatrices = inst.getOutputMatrices();
- boolean execCP = false;
-
- // Spawn MapReduce Jobs
- try {
- // replace all placeholders in all instructions with appropriate values
- String rdInst = inst.getIv_randInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String shuffleInst = inst.getIv_shuffleInstructions();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
- boolean jvmReuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
-
- switch(inst.getJobType()) {
-
- case GMR:
- case GMRCELL:
- ret = GMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- inst.getPartitioned(), inst.getPformats(), inst.getPsizes(),
- rrInst, mapInst, aggInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), jvmReuse, inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case DATAGEN:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPDataGen( inst, rdInst ) )
- {
- ret = executeInMemoryDataGenOperations(inst, rdInst, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- ret = DataGenMR.runJob(inst,
- rdInst.split(Lop.INSTRUCTION_DELIMITOR), mapInst, aggInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos());
- }
- break;
-
- case CM_COV:
- ret = CMCOVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case GROUPED_AGG:
- ret = GroupedAggMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case REBLOCK:
- case CSV_REBLOCK:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPReblock( inst, inputMatrices ) )
- {
- ret = executeInMemoryReblockOperations(inst, shuffleInst, inputMatrices, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- // export dirty matrices to HDFS (initially deferred)
- for(MatrixObject m : inputMatrices) {
- if ( m.isDirty() )
- m.exportData();
- }
- checkEmptyInputs( inst, inputMatrices );
-
- if ( inst.getJobType() == JobType.REBLOCK ) {
- ret = ReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(), getNNZ(inputMatrices),
- mapInst, shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), jvmReuse, inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- }
- else if( inst.getJobType() == JobType.CSV_REBLOCK ) {
- ret = CSVReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- }
- }
- break;
-
- case CSV_WRITE:
- ret = WriteCSVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBclens(), inst.getBclens(), shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getOutputs());
- break;
-
- case MMCJ:
- ret = MMCJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, aggInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(),
- inst.getOutputs()[0], inst.getOutputInfos()[0] );
- break;
-
- case MMRJ:
- ret = MMRJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, aggInst, shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case SORT:
- boolean weightsflag = true;
- if ( !mapInst.equalsIgnoreCase("") )
- weightsflag = false;
- ret = SortMR.runJob(inst, inst.getInputs()[0], inst.getInputInfos()[0],
- inst.getRlens()[0], inst.getClens()[0], inst.getBrlens()[0], inst.getBclens()[0],
- mapInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(),
- inst.getOutputs()[0], inst.getOutputInfos()[0], weightsflag );
- break;
-
- case COMBINE:
- ret = CombineMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case DATA_PARTITION:
- ret = DataPartitionMR.runJob(inst, inputMatrices, shuffleInst, inst.getIv_resultIndices(), outputMatrices, inst.getIv_numReducers(), inst.getIv_replication());
- break;
-
- case TRANSFORM:
-
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && OptimizerUtils.ALLOW_TRANSFORM_RECOMPILE
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPTransform( inst, inputMatrices ) )
- {
- // transform the data and generate output in CSV format
- ret = executeInMemoryTransform(inst, inputMatrices, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- ret = DataTransform.mrDataTransform(inst, inputMatrices, shuffleInst, otherInst, inst.getIv_resultIndices(), outputMatrices, inst.getIv_numReducers(), inst.getIv_replication());
- }
- break;
-
- default:
- throw new DMLRuntimeException("Invalid jobtype: " + inst.getJobType());
- }
-
- } // end of try block
- catch (Exception e) {
- throw new DMLRuntimeException( e );
- }
-
- if (ret.checkReturnStatus()) {
- /*
- * Check if any output is empty. If yes, create a dummy file. Needs
- * to be done only in case of (1) CellOutputInfo and if not CP, or
- * (2) BinaryBlockOutputInfo if not CP and output empty blocks disabled.
- */
- try {
- if( !execCP )
- {
- for (int i = 0; i < outputMatrices.length; i++) {
- //get output meta data
- MatrixFormatMetaData meta = (MatrixFormatMetaData)outputMatrices[i].getMetaData();
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- OutputInfo outinfo = meta.getOutputInfo();
- String fname = outputMatrices[i].getFileName();
-
- if (MapReduceTool.isHDFSFileEmpty(fname))
- {
- //prepare output file
- Path filepath = new Path(fname, "0-m-00000");
- MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(outinfo);
- writer.writeEmptyMatrixToHDFS(filepath.toString(), mc.getRows(), mc.getCols(),
- mc.getRowsPerBlock(), mc.getColsPerBlock());
- }
-
- outputMatrices[i].setFileExists(true);
-
- if ( inst.getJobType() != JobType.CSV_WRITE && inst.getJobType() != JobType.TRANSFORM) {
- // write out metadata file
- // Currently, valueType information in not stored in MR instruction,
- // since only DOUBLE matrices are supported ==> hard coded the value type information for now
- MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE, ((MatrixDimensionsMetaData)ret.getMetaData(i)).getMatrixCharacteristics(), outinfo);
- }
- }
- }
- return ret;
- } catch (IOException e) {
- throw new DMLRuntimeException(e);
- }
- }
-
- // should not come here!
- throw new DMLRuntimeException("Unexpected Job Type: " + inst.getJobType());
- }
-
- /**
- *
- * @param inst
- * @param inputMatrices
- * @param pb
- * @throws DMLRuntimeException
- */
- private static void checkEmptyInputs( MRJobInstruction inst, MatrixObject[] inputMatrices )
- throws DMLRuntimeException
- {
- // Check if any of the input files are empty.. only for those job types
- // for which empty inputs are NOT allowed
- if (!inst.getJobType().areEmptyInputsAllowed()) {
- for ( int i=0; i < inputMatrices.length; i++ ) {
- try {
- if (MapReduceTool.isHDFSFileEmpty(inputMatrices[i].getFileName())) {
- throw new DMLRuntimeException( "Can not operate on an empty file: " + inputMatrices[i].getFileName());
- }
- } catch (IOException e) {
- throw new DMLRuntimeException( "runtime error occurred -- " , e);
- }
- }
- }
- }
-
- /**
- * Computes the replacement string for a given variable name placeholder string
- * (e.g., ##mVar2## or ##Var5##). The replacement is a HDFS filename for matrix
- * variables, and is the actual value (stored in symbol table) for scalar variables.
- *
- * @param inst
- * @param varName
- * @param map
- * @return
- * @throws DMLRuntimeException
- */
- private static String getVarNameReplacement(String inst, String varName, LocalVariableMap map) throws DMLRuntimeException {
- Data val = map.get(varName);
- if ( val != null ) {
- String replacement = null;
- if (val.getDataType() == DataType.MATRIX) {
- replacement = ((MatrixObject)val).getFileName();
- }
-
- if (val.getDataType() == DataType.SCALAR)
- replacement = "" + ((ScalarObject) val).getStringValue();
- return replacement;
- }
- else {
- throw new DMLRuntimeException("Variable ("+varName+") in Instruction ("+inst+") is not found in the variablemap.");
- }
- }
-
- /**
- * Replaces ALL placeholder strings (such as ##mVar2## and ##Var5##) in a single instruction.
- *
- * @param inst
- * @param map
- * @return
- * @throws DMLRuntimeException
- */
- private static String updateInstLabels(String inst, LocalVariableMap map) throws DMLRuntimeException {
- if ( inst.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
- int skip = Lop.VARIABLE_NAME_PLACEHOLDER.toString().length();
- while ( inst.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
- int startLoc = inst.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER)+skip;
- String varName = inst.substring(startLoc, inst.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, startLoc));
- String replacement = getVarNameReplacement(inst, varName, map);
- inst = inst.replaceAll(Lop.VARIABLE_NAME_PLACEHOLDER + varName + Lop.VARIABLE_NAME_PLACEHOLDER, replacement);
- }
- }
- return inst;
- }
-
- /**
- * Takes a delimited string of instructions, and replaces ALL placeholder labels
- * (such as ##mVar2## and ##Var5##) in ALL instructions.
- *
- * @param instList
- * @param labelValueMapping
- * @return
- * @throws DMLRuntimeException
- */
- public static String updateLabels (String instList, LocalVariableMap labelValueMapping) throws DMLRuntimeException {
-
- if ( !instList.contains(Lop.VARIABLE_NAME_PLACEHOLDER) )
- return instList;
-
- StringBuilder updateInstList = new StringBuilder();
- String[] ilist = instList.split(Lop.INSTRUCTION_DELIMITOR);
-
- for ( int i=0; i < ilist.length; i++ ) {
- if ( i > 0 )
- updateInstList.append(Lop.INSTRUCTION_DELIMITOR);
-
- updateInstList.append( updateInstLabels(ilist[i], labelValueMapping));
- }
- return updateInstList.toString();
- }
-
-
- /**
- *
- * @param inputMatrices
- * @return
- * @throws DMLRuntimeException
- */
- private static long[] getNNZ( MatrixObject[] inputMatrices )
- throws DMLRuntimeException
- {
- int len = inputMatrices.length;
- long[] ret = new long[len];
- for( int i=0; i<len; i++ )
- {
- MatrixObject mo = inputMatrices[i];
- if( mo != null )
- ret[i] = mo.getNnz();
- else
- ret[i] = -1;
- }
-
- return ret;
- }
-
- /**
- *
- * @param inst
- * @param shuffleInst
- * @param inputMatrices
- * @param outputMatrices
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- private static JobReturn executeInMemoryReblockOperations( MRJobInstruction inst, String shuffleInst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
- ReblockInstruction[] rblkSet = MRInstructionParser.parseReblockInstructions(shuffleInst);
- byte[] results = inst.getIv_resultIndices();
- for( ReblockInstruction rblk : rblkSet )
- {
- //CP Reblock through caching framework (no copy required: same data, next op copies)
- MatrixBlock mb = inputMatrices[rblk.input].acquireRead();
- for( int i=0; i<results.length; i++ )
- if( rblk.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), rblk.brlen, rblk.bclen, mb.getNonZeros());
- }
- inputMatrices[rblk.input].release();
- }
-
- return new JobReturn( mc, inst.getOutputInfos(), true);
- }
-
- private static JobReturn executeInMemoryDataGenOperations( MRJobInstruction inst, String randInst, MatrixObject[] outputMatrices )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
- DataGenMRInstruction[] dgSet = MRInstructionParser.parseDataGenInstructions(randInst);
- byte[] results = inst.getIv_resultIndices();
- for( DataGenMRInstruction ldgInst : dgSet )
- {
- if( ldgInst instanceof RandInstruction )
- {
- //CP Rand block operation
- RandInstruction lrand = (RandInstruction)ldgInst;
- RandomMatrixGenerator rgen = LibMatrixDatagen.createRandomMatrixGenerator(
- lrand.getProbabilityDensityFunction(),
- (int)lrand.getRows(), (int)lrand.getCols(),
- lrand.getRowsInBlock(), lrand.getColsInBlock(),
- lrand.getSparsity(), lrand.getMinValue(), lrand.getMaxValue(),
- lrand.getPdfParams());
-
-
- MatrixBlock mb = MatrixBlock.randOperations(rgen, lrand.getSeed());
-
- for( int i=0; i<results.length; i++ )
- if( lrand.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), lrand.getRowsInBlock(), lrand.getColsInBlock(), mb.getNonZeros());
- }
- }
- else if( ldgInst instanceof SeqInstruction )
- {
- SeqInstruction lseq = (SeqInstruction) ldgInst;
- MatrixBlock mb = MatrixBlock.seqOperations(lseq.fromValue, lseq.toValue, lseq.incrValue);
- for( int i=0; i<results.length; i++ )
- if( lseq.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), lseq.getRowsInBlock(), lseq.getColsInBlock(), mb.getNonZeros());
- }
- }
- }
-
- return new JobReturn( mc, inst.getOutputInfos(), true);
- }
-
- private static JobReturn executeInMemoryTransform( MRJobInstruction inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) throws IOException, DMLRuntimeException, IllegalArgumentException, JSONException {
- return DataTransform.cpDataTransform(
- inst.getIv_shuffleInstructions(),
- inputMatrices,
- outputMatrices);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java b/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
deleted file mode 100644
index d392357..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.antlr4.DMLParserWrapper;
-import com.ibm.bi.dml.parser.python.PyDMLParserWrapper;
-
-/**
- * Base class for all dml parsers in order to make the various compilation chains
- * independent of the used parser.
- */
-public abstract class AParserWrapper
-{
- /**
- *
- * @param fileName
- * @param dmlScript
- * @param argVals
- * @return
- * @throws ParseException
- */
- public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
- throws ParseException;
-
-
- /**
- * Factory method for creating instances of AParserWrapper, for
- * simplificy fused with the abstract class.
- *
- * @param pydml
- * @return
- */
- public static AParserWrapper createParser(boolean pydml)
- {
- AParserWrapper ret = null;
-
- //create the parser instance
- if( pydml )
- ret = new PyDMLParserWrapper();
- else
- ret = new DMLParserWrapper();
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java b/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
deleted file mode 100644
index 870cbe3..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-
-
-public class AssignmentStatement extends Statement
-{
-
- private ArrayList<DataIdentifier> _targetList;
- private Expression _source;
-
- // rewrites statement to support function inlining (creates deep copy)
- public Statement rewriteStatement(String prefix) throws LanguageException{
-
- // rewrite target (deep copy)
- DataIdentifier newTarget = (DataIdentifier)_targetList.get(0).rewriteExpression(prefix);
-
- // rewrite source (deep copy)
- Expression newSource = _source.rewriteExpression(prefix);
-
- // create rewritten assignment statement (deep copy)
- AssignmentStatement retVal = new AssignmentStatement(newTarget, newSource,this.getBeginLine(),
- this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- return retVal;
- }
-
-
- public AssignmentStatement(DataIdentifier t, Expression s) {
-
- _targetList = new ArrayList<DataIdentifier>();
- _targetList.add(t);
- _source = s;
- }
-
-
- public AssignmentStatement(DataIdentifier t, Expression s, int beginLine, int beginCol, int endLine, int endCol)
- throws LanguageException
- {
- _targetList = new ArrayList<DataIdentifier>();
- _targetList.add(t);
- _source = s;
-
- setBeginLine(beginLine);
- setBeginColumn(beginCol);
- setEndLine(endLine);
- setEndColumn(endCol);
-
- }
-
- public DataIdentifier getTarget(){
- return _targetList.get(0);
- }
-
- public ArrayList<DataIdentifier> getTargetList()
- {
- return _targetList;
- }
-
- public Expression getSource(){
- return _source;
- }
- public void setSource(Expression s){
- _source = s;
- }
-
- @Override
- public boolean controlStatement() {
- // ensure that breakpoints end up in own statement block
- if (DMLScript.ENABLE_DEBUG_MODE) {
- DMLBreakpointManager.insertBreakpoint(_source.getBeginLine());
- return true;
- }
-
- // for now, ensure that function call ends up in different statement block
- if (_source instanceof FunctionCallIdentifier)
- return true;
-
- return false;
- }
-
- public void initializeforwardLV(VariableSet activeIn){
- //do nothing
- }
-
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- // add variables read by source expression
- result.addVariables(_source.variablesRead());
-
- // for LHS IndexedIdentifier, add variables for indexing expressions
- for (int i=0; i<_targetList.size(); i++){
- if (_targetList.get(i) instanceof IndexedIdentifier) {
- IndexedIdentifier target = (IndexedIdentifier) _targetList.get(i);
- result.addVariables(target.variablesRead());
- }
- }
- return result;
- }
-
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
-
- // add target to updated list
- for (DataIdentifier target : _targetList)
- result.addVariable(target.getName(), target);
- return result;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- for (int i=0; i< _targetList.size(); i++){
- sb.append(_targetList.get(i).toString());
- }
- sb.append(" = ");
- sb.append(_source.toString());
- sb.append(";");
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java b/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
deleted file mode 100644
index 38f7fb9..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-
-public class BinaryExpression extends Expression
-{
-
- private Expression _left;
- private Expression _right;
- private BinaryOp _opcode;
-
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
-
-
- BinaryExpression newExpr = new BinaryExpression(this._opcode,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- newExpr.setLeft(_left.rewriteExpression(prefix));
- newExpr.setRight(_right.rewriteExpression(prefix));
- return newExpr;
- }
-
- public BinaryExpression(BinaryOp bop) {
- _kind = Kind.BinaryOp;
- _opcode = bop;
-
- setFilename("MAIN SCRIPT");
- setBeginLine(0);
- setBeginColumn(0);
- setEndLine(0);
- setEndColumn(0);
- }
-
- public BinaryExpression(BinaryOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn) {
- _kind = Kind.BinaryOp;
- _opcode = bop;
-
- setFilename(filename);
- setBeginLine(beginLine);
- setBeginColumn(beginColumn);
- setEndLine(endLine);
- setEndColumn(endColumn);
- }
-
-
- public BinaryOp getOpCode() {
- return _opcode;
- }
-
- public void setLeft(Expression l) {
- _left = l;
-
- // update script location information --> left expression is BEFORE in script
- if (_left != null){
- setFilename(_left.getFilename());
- setBeginLine(_left.getBeginLine());
- setBeginColumn(_left.getBeginColumn());
- }
-
- }
-
- public void setRight(Expression r) {
- _right = r;
-
- // update script location information --> right expression is AFTER in script
- if (_right != null){
- setFilename(_right.getFilename());
- setBeginLine(_right.getEndLine());
- setBeginColumn(_right.getEndColumn());
- }
- }
-
- public Expression getLeft() {
- return _left;
- }
-
- public Expression getRight() {
- return _right;
- }
-
- /**
- * Validate parse tree : Process Binary Expression in an assignment
- * statement
- *
- * @throws LanguageException
- */
- @Override
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- //recursive validate
- if (_left instanceof FunctionCallIdentifier || _right instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in binary expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- _left.validateExpression(ids, constVars, conditional);
- _right.validateExpression(ids, constVars, conditional);
-
- //constant propagation (precondition for more complex constant folding rewrite)
- if( _left instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _left).getName()) )
- _left = constVars.get(((DataIdentifier) _left).getName());
- if( _right instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _right).getName()) )
- _right = constVars.get(((DataIdentifier) _right).getName());
-
-
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- output.setDataType(computeDataType(this.getLeft(), this.getRight(), true));
- ValueType resultVT = computeValueType(this.getLeft(), this.getRight(), true);
-
- // Override the computed value type, if needed
- if (this.getOpCode() == Expression.BinaryOp.POW
- || this.getOpCode() == Expression.BinaryOp.DIV) {
- resultVT = ValueType.DOUBLE;
- }
-
- output.setValueType(resultVT);
-
- checkAndSetDimensions(output, conditional);
- if (this.getOpCode() == Expression.BinaryOp.MATMULT) {
- if ((this.getLeft().getOutput().getDataType() != DataType.MATRIX) || (this.getRight().getOutput().getDataType() != DataType.MATRIX)) {
- // remove exception for now
- // throw new LanguageException(
- // "Matrix multiplication not supported for scalars",
- // LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if (this.getLeft().getOutput().getDim2() != -1
- && this.getRight().getOutput().getDim1() != -1
- && this.getLeft().getOutput().getDim2() != this.getRight()
- .getOutput().getDim1())
- {
- raiseValidateError("invalid dimensions for matrix multiplication (k1="+this.getLeft().getOutput().getDim2()+", k2="+this.getRight().getOutput().getDim1()+")",
- conditional, LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- output.setDimensions(this.getLeft().getOutput().getDim1(), this
- .getRight().getOutput().getDim2());
- }
-
- this.setOutput(output);
- }
-
- private void checkAndSetDimensions(DataIdentifier output, boolean conditional)
- throws LanguageException {
- Identifier left = this.getLeft().getOutput();
- Identifier right = this.getRight().getOutput();
- Identifier pivot = null;
- Identifier aux = null;
-
- if (left.getDataType() == DataType.MATRIX) {
- pivot = left;
- if (right.getDataType() == DataType.MATRIX) {
- aux = right;
- }
- } else if (right.getDataType() == DataType.MATRIX) {
- pivot = right;
- }
-
- if ((pivot != null) && (aux != null))
- {
- //check dimensions binary operations (if dims known)
- if (isSameDimensionBinaryOp(this.getOpCode()) && pivot.dimsKnown() && aux.dimsKnown() )
- {
- if( (pivot.getDim1() != aux.getDim1() && aux.getDim1()>1) //number of rows must always be equivalent if not row vector
- || (pivot.getDim2() != aux.getDim2() && aux.getDim2()>1)) //number of cols must be equivalent if not col vector
- {
- raiseValidateError("Mismatch in dimensions for operation "+ this.toString(), conditional, LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- //set dimension information
- if (pivot != null) {
- output.setDimensions(pivot.getDim1(), pivot.getDim2());
- }
- }
-
- public String toString() {
-
- return "(" + _left.toString() + " " + _opcode.toString() + " "
- + _right.toString() + ")";
-
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesRead());
- result.addVariables(_right.variablesRead());
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesUpdated());
- result.addVariables(_right.variablesUpdated());
- return result;
- }
-
- public static boolean isSameDimensionBinaryOp(BinaryOp op) {
- return (op == BinaryOp.PLUS) || (op == BinaryOp.MINUS)
- || (op == BinaryOp.MULT) || (op == BinaryOp.DIV)
- || (op == BinaryOp.MODULUS) || (op == BinaryOp.INTDIV)
- || (op == BinaryOp.POW);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java b/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
deleted file mode 100644
index 30f154d..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-
-public class BooleanExpression extends Expression
-{
-
- private Expression _left;
- private Expression _right;
- private BooleanOp _opcode;
-
- public BooleanExpression(BooleanOp bop){
- _kind = Kind.BooleanOp;
- _opcode = bop;
-
- setFilename("MAIN SCRIPT");
- setBeginLine(0);
- setBeginColumn(0);
- setEndLine(0);
- setEndColumn(0);
- }
-
- public BooleanExpression(BooleanOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn){
- _kind = Kind.BooleanOp;
- _opcode = bop;
-
- setFilename(filename);
- setBeginLine(beginLine);
- setBeginColumn(beginColumn);
- setEndLine(endLine);
- setEndColumn(endColumn);
- }
-
- public BooleanOp getOpCode(){
- return _opcode;
- }
-
- public void setLeft(Expression l){
- _left = l;
-
- // update script location information --> left expression is BEFORE in script
- if (_left != null){
- this.setFilename(_left.getFilename());
- this.setBeginLine(_left.getBeginLine());
- this.setBeginColumn(_left.getBeginColumn());
- }
- }
-
- public void setRight(Expression r){
- _right = r;
-
- // update script location information --> right expression is AFTER in script
- if (_right != null){
- this.setFilename(_right.getFilename());
- this.setBeginLine(_right.getBeginLine());
- this.setBeginColumn(_right.getBeginColumn());
- }
- }
-
- public Expression getLeft(){
- return _left;
- }
-
- public Expression getRight(){
- return _right;
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
-
-
- BooleanExpression newExpr = new BooleanExpression(this._opcode, this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- newExpr.setLeft(_left.rewriteExpression(prefix));
- if (_right != null)
- newExpr.setRight(_right.rewriteExpression(prefix));
- return newExpr;
- }
-
- /**
- * Validate parse tree : Process Boolean Expression
- */
- @Override
- public void validateExpression(HashMap<String,DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional) throws LanguageException{
-
- this.getLeft().validateExpression(ids, constVars, conditional);
-
- //recursive validate
- if (_left instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in boolean expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- if (this.getRight() != null) {
-
- if (_right instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in boolean expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- this.getRight().validateExpression(ids, constVars, conditional);
- }
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- output.setBooleanProperties();
- this.setOutput(output);
- if ((_opcode == Expression.BooleanOp.CONDITIONALAND) ||
- (_opcode == Expression.BooleanOp.CONDITIONALOR))
- { //always unconditional (because unsupported operation)
- raiseValidateError("Unsupported boolean operation " + _opcode.toString(), false, LanguageException.LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- public String toString(){
- if (_opcode == BooleanOp.NOT) {
- return "(" + _opcode.toString() + " " + _left.toString() + ")";
- } else {
- return "(" + _left.toString() + " " + _opcode.toString() + " " + _right.toString() + ")";
- }
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesRead());
- if (_right != null){
- result.addVariables(_right.variablesRead());
- }
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesUpdated());
- if (_right != null){
- result.addVariables(_right.variablesUpdated());
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
deleted file mode 100644
index ac5cb92..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class BooleanIdentifier extends ConstIdentifier
-{
-
- private boolean _val;
-
-
- public BooleanIdentifier(boolean val, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = val;
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.BOOLEAN);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- return this;
- }
-
- public boolean getValue(){
- return _val;
- }
-
- public String toString(){
- return Boolean.toString(_val);
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public long getLongValue(){
- return (getValue() ? 1 : 0);
- }
-}
[22/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java b/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
deleted file mode 100644
index 2f39797..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
+++ /dev/null
@@ -1,1425 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-
-public class BuiltinFunctionExpression extends DataIdentifier
-{
-
- protected Expression[] _args = null;
- private BuiltinFunctionOp _opcode;
-
- public BuiltinFunctionExpression(BuiltinFunctionOp bifop, ArrayList<ParameterExpression> args, String fname, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.BuiltinFunctionOp;
- _opcode = bifop;
- _args = new Expression[args.size()];
- for(int i=0; i < args.size(); i++) {
- _args[i] = args.get(i).getExpr();
- }
- this.setAllPositions(fname, blp, bcp, elp, ecp);
- }
-
- public BuiltinFunctionExpression(BuiltinFunctionOp bifop, Expression[] args, String fname, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.BuiltinFunctionOp;
- _opcode = bifop;
- _args = new Expression[args.length];
- for(int i=0; i < args.length; i++) {
- _args[i] = args[i];
- }
- this.setAllPositions(fname, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- Expression[] newArgs = new Expression[_args.length];
- for(int i=0; i < _args.length; i++) {
- newArgs[i] = _args[i].rewriteExpression(prefix);
- }
- BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(this._opcode, newArgs,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- return retVal;
-
- }
-
- public BuiltinFunctionOp getOpCode() {
- return _opcode;
- }
-
- public Expression getFirstExpr() {
- return (_args.length >= 1 ? _args[0] : null);
- }
-
- public Expression getSecondExpr() {
- return (_args.length >= 2 ? _args[1] : null);
- }
-
- public Expression getThirdExpr() {
- return (_args.length >= 3 ? _args[2] : null);
- }
-
- public Expression[] getAllExpr(){
- return _args;
- }
-
- @Override
- public void validateExpression(MultiAssignmentStatement stmt, HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- if (this.getFirstExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
-
- this.getFirstExpr().validateExpression(ids, constVars, conditional);
- if (getSecondExpr() != null){
- if (this.getSecondExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
- getSecondExpr().validateExpression(ids, constVars, conditional);
- }
- if (getThirdExpr() != null) {
- if (this.getThirdExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
- getThirdExpr().validateExpression(ids, constVars, conditional);
- }
- _outputs = new Identifier[stmt.getTargetList().size()];
- int count = 0;
- for (DataIdentifier outParam: stmt.getTargetList()){
- DataIdentifier tmp = new DataIdentifier(outParam);
- tmp.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- _outputs[count++] = tmp;
- }
-
- switch (_opcode) {
- case QR:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier qrOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier qrOut2 = (DataIdentifier) getOutputs()[1];
-
- long rows = getFirstExpr().getOutput().getDim1();
- long cols = getFirstExpr().getOutput().getDim2();
-
- // Output1 - Q
- qrOut1.setDataType(DataType.MATRIX);
- qrOut1.setValueType(ValueType.DOUBLE);
- qrOut1.setDimensions(rows, cols);
- qrOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - R
- qrOut2.setDataType(DataType.MATRIX);
- qrOut2.setValueType(ValueType.DOUBLE);
- qrOut2.setDimensions(rows, cols);
- qrOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- case LU:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier luOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier luOut2 = (DataIdentifier) getOutputs()[1];
- DataIdentifier luOut3 = (DataIdentifier) getOutputs()[2];
-
- long inrows = getFirstExpr().getOutput().getDim1();
- long incols = getFirstExpr().getOutput().getDim2();
-
- if ( inrows != incols ) {
- raiseValidateError("LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + inrows + ", cols="+incols+")", conditional);
- }
-
- // Output1 - P
- luOut1.setDataType(DataType.MATRIX);
- luOut1.setValueType(ValueType.DOUBLE);
- luOut1.setDimensions(inrows, inrows);
- luOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - L
- luOut2.setDataType(DataType.MATRIX);
- luOut2.setValueType(ValueType.DOUBLE);
- luOut2.setDimensions(inrows, inrows);
- luOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output3 - U
- luOut3.setDataType(DataType.MATRIX);
- luOut3.setValueType(ValueType.DOUBLE);
- luOut3.setDimensions(inrows, inrows);
- luOut3.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- case EIGEN:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier eigenOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier eigenOut2 = (DataIdentifier) getOutputs()[1];
-
- if ( getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2() ) {
- raiseValidateError("Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + getFirstExpr().getOutput().getDim1() + ", cols="+ getFirstExpr().getOutput().getDim2() +")", conditional);
- }
-
- // Output1 - Eigen Values
- eigenOut1.setDataType(DataType.MATRIX);
- eigenOut1.setValueType(ValueType.DOUBLE);
- eigenOut1.setDimensions(getFirstExpr().getOutput().getDim1(), 1);
- eigenOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - Eigen Vectors
- eigenOut2.setDataType(DataType.MATRIX);
- eigenOut2.setValueType(ValueType.DOUBLE);
- eigenOut2.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2());
- eigenOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- default: //always unconditional
- raiseValidateError("Unknown Builtin Function opcode: " + _opcode, false);
- }
- }
-
- /**
- * Validate parse tree : Process BuiltinFunction Expression in an assignment
- * statement
- *
- * @throws LanguageException
- */
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException {
-
- for(int i=0; i < _args.length; i++ ) {
-
- if (_args[i] instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
-
- _args[i].validateExpression(ids, constVars, conditional);
- }
-
- // checkIdentifierParams();
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- Identifier id = this.getFirstExpr().getOutput();
- output.setProperties(this.getFirstExpr().getOutput());
- output.setNnz(-1); //conservatively, cannot use input nnz!
- this.setOutput(output);
-
- switch (this.getOpCode()) {
- case COLSUM:
- case COLMAX:
- case COLMIN:
- case COLMEAN:
- // colSums(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(1, id.getDim2());
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case ROWSUM:
- case ROWMAX:
- case ROWINDEXMAX:
- case ROWMIN:
- case ROWINDEXMIN:
- case ROWMEAN:
- //rowSums(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim1(), 1);
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case SUM:
- case PROD:
- case TRACE:
- // sum(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
-
- break;
-
- case MEAN:
- //checkNumParameters(2, false); // mean(Y) or mean(Y,W)
- if (getSecondExpr() != null) {
- checkNumParameters (2);
- }
- else {
- checkNumParameters (1);
- }
-
- checkMatrixParam(getFirstExpr());
- if ( getSecondExpr() != null ) {
- // x = mean(Y,W);
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
- break;
-
- case MIN:
- case MAX:
- //min(X), min(X,s), min(s,X), min(s,r), min(X,Y)
-
- //unary aggregate
- if (getSecondExpr() == null)
- {
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType( DataType.SCALAR );
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- }
- //binary operation
- else
- {
- checkNumParameters(2);
- DataType dt1 = getFirstExpr().getOutput().getDataType();
- DataType dt2 = getSecondExpr().getOutput().getDataType();
- DataType dtOut = (dt1==DataType.MATRIX || dt2==DataType.MATRIX)?
- DataType.MATRIX : DataType.SCALAR;
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX )
- checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true);
- //determine output dimensions
- long[] dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr());
- output.setDataType( dtOut );
- output.setDimensions(dims[0], dims[1]);
- output.setBlockDimensions (dims[2], dims[3]);
- }
- output.setValueType(id.getValueType());
-
- break;
-
- case CUMSUM:
- case CUMPROD:
- case CUMMIN:
- case CUMMAX:
- // cumsum(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim1(), id.getDim2());
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
-
- break;
-
- case CAST_AS_SCALAR:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- if (( getFirstExpr().getOutput().getDim1() != -1 && getFirstExpr().getOutput().getDim1() !=1) || ( getFirstExpr().getOutput().getDim2() != -1 && getFirstExpr().getOutput().getDim2() !=1)) {
- raiseValidateError("dimension mismatch while casting matrix to scalar: dim1: " + getFirstExpr().getOutput().getDim1() + " dim2 " + getFirstExpr().getOutput().getDim2(),
- conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
- break;
- case CAST_AS_MATRIX:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(1, 1);
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case CAST_AS_DOUBLE:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.DOUBLE);
- break;
- case CAST_AS_INT:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.INT);
- break;
- case CAST_AS_BOOLEAN:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.BOOLEAN);
- break;
-
- case CBIND:
- case RBIND:
- checkNumParameters(2);
-
- //scalar string append (string concatenation with \n)
- if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) {
- checkScalarParam(getFirstExpr());
- checkScalarParam(getSecondExpr());
- checkValueTypeParam(getFirstExpr(), ValueType.STRING);
- checkValueTypeParam(getSecondExpr(), ValueType.STRING);
- }
- //matrix append (rbind/cbind)
- else {
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- }
-
- output.setDataType(id.getDataType());
- output.setValueType(id.getValueType());
-
- // set output dimensions and validate consistency
- long appendDim1 = -1, appendDim2 = -1;
- long m1rlen = getFirstExpr().getOutput().getDim1();
- long m1clen = getFirstExpr().getOutput().getDim2();
- long m2rlen = getSecondExpr().getOutput().getDim1();
- long m2clen = getSecondExpr().getOutput().getDim2();
-
- if( getOpCode() == BuiltinFunctionOp.CBIND ) {
- if (m1rlen > 0 && m2rlen > 0 && m1rlen!=m2rlen) {
- raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " +
- m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0) ? m1rlen : m2rlen;
- appendDim2 = (m1clen>0 && m2clen>0)? m1clen + m2clen : -1;
- }
- else if( getOpCode() == BuiltinFunctionOp.RBIND ) {
- if (m1clen > 0 && m2clen > 0 && m1clen!=m2clen) {
- raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " +
- m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0 && m2rlen>0)? m1rlen + m2rlen : -1;
- appendDim2 = (m1clen>0) ? m1clen : m2clen;
- }
-
- output.setDimensions(appendDim1, appendDim2);
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
-
- break;
-
- case PPRED:
- // ppred (X,Y, "<"); ppred (X,y, "<"); ppred (y,X, "<");
- checkNumParameters(3);
-
- DataType dt1 = getFirstExpr().getOutput().getDataType();
- DataType dt2 = getSecondExpr().getOutput().getDataType();
-
- //check input data types
- if( dt1 == DataType.SCALAR && dt2 == DataType.SCALAR ) {
- raiseValidateError("ppred() requires at least one matrix input.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if( dt1 == DataType.MATRIX )
- checkMatrixParam(getFirstExpr());
- if( dt2 == DataType.MATRIX )
- checkMatrixParam(getSecondExpr());
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX ) //dt1==dt2
- checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true);
-
- //check operator
- if (getThirdExpr().getOutput().getDataType() != DataType.SCALAR ||
- getThirdExpr().getOutput().getValueType() != ValueType.STRING)
- {
- raiseValidateError("Third argument in ppred() is not an operator ", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- //determine output dimensions
- long[] dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(dims[0], dims[1]);
- output.setBlockDimensions(dims[2], dims[3]);
- output.setValueType(id.getValueType());
- break;
-
- case TRANS:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim2(), id.getDim1());
- output.setBlockDimensions (id.getColumnsInBlock(), id.getRowsInBlock());
- output.setValueType(id.getValueType());
- break;
- case DIAG:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- if( id.getDim2() != -1 ) { //type known
- if ( id.getDim2() == 1 )
- {
- //diag V2M
- output.setDimensions(id.getDim1(), id.getDim1());
- }
- else
- {
- if (id.getDim1() != id.getDim2()) {
- raiseValidateError("Invoking diag on matrix with dimensions ("
- + id.getDim1() + "," + id.getDim2()
- + ") in " + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- //diag M2V
- output.setDimensions(id.getDim1(), 1);
- }
- }
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case NROW:
- case NCOL:
- case LENGTH:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.INT);
- break;
-
- // Contingency tables
- case TABLE:
-
- /*
- * Allowed #of arguments: 2,3,4,5
- * table(A,B)
- * table(A,B,W)
- * table(A,B,1)
- * table(A,B,dim1,dim2)
- * table(A,B,W,dim1,dim2)
- * table(A,B,1,dim1,dim2)
- */
-
- // Check for validity of input arguments, and setup output dimensions
-
- // First input: is always of type MATRIX
- checkMatrixParam(getFirstExpr());
-
- if ( getSecondExpr() == null )
- raiseValidateError("Invalid number of arguments to table(): "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
-
- // Second input: can be MATRIX or SCALAR
- // cases: table(A,B) or table(A,1)
- if ( getSecondExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
-
- long outputDim1=-1, outputDim2=-1;
-
- switch(_args.length) {
- case 2:
- // nothing to do
- break;
-
- case 3:
- // case - table w/ weights
- // - weights specified as a matrix: table(A,B,W) or table(A,1,W)
- // - weights specified as a scalar: table(A,B,1) or table(A,1,1)
- if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getThirdExpr());
- break;
-
- case 4:
- // case - table w/ output dimensions: table(A,B,dim1,dim2) or table(A,1,dim1,dim2)
- // third and fourth arguments must be scalars
- if ( getThirdExpr().getOutput().getDataType() != DataType.SCALAR || _args[3].getOutput().getDataType() != DataType.SCALAR ) {
- raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else {
- // constant propagation
- if( getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) )
- _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName());
- if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) )
- _args[3] = constVars.get(((DataIdentifier)_args[3]).getName());
-
- if ( getThirdExpr().getOutput() instanceof ConstIdentifier )
- outputDim1 = ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue();
- if ( _args[3].getOutput() instanceof ConstIdentifier )
- outputDim2 = ((ConstIdentifier) _args[3].getOutput()).getLongValue();
- }
- break;
-
- case 5:
- // case - table w/ weights and output dimensions:
- // - table(A,B,W,dim1,dim2) or table(A,1,W,dim1,dim2)
- // - table(A,B,1,dim1,dim2) or table(A,1,1,dim1,dim2)
-
- if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getThirdExpr());
-
- // fourth and fifth arguments must be scalars
- if ( _args[3].getOutput().getDataType() != DataType.SCALAR || _args[4].getOutput().getDataType() != DataType.SCALAR ) {
- raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else {
- // constant propagation
- if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) )
- _args[3] = constVars.get(((DataIdentifier)_args[3]).getName());
- if( _args[4] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[4]).getName()) )
- _args[4] = constVars.get(((DataIdentifier)_args[4]).getName());
-
- if ( _args[3].getOutput() instanceof ConstIdentifier )
- outputDim1 = ((ConstIdentifier) _args[3].getOutput()).getLongValue();
- if ( _args[4].getOutput() instanceof ConstIdentifier )
- outputDim2 = ((ConstIdentifier) _args[4].getOutput()).getLongValue();
- }
- break;
-
- default:
- raiseValidateError("Invalid number of arguments to table(): "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- // The dimensions for the output matrix will be known only at the
- // run time
- output.setDimensions(outputDim1, outputDim2);
- output.setBlockDimensions (-1, -1);
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- break;
-
- case MOMENT:
- /*
- * x = centralMoment(V,order) or xw = centralMoment(V,W,order)
- */
- checkMatrixParam(getFirstExpr());
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
- checkScalarParam(getThirdExpr());
- }
- else {
- checkNumParameters(2);
- checkScalarParam(getSecondExpr());
- }
-
- // output is a scalar
- output.setDataType(DataType.SCALAR);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- break;
-
- case COV:
- /*
- * x = cov(V1,V2) or xw = cov(V1,V2,W)
- */
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
-
- if (getThirdExpr() != null) {
- checkMatrixParam(getThirdExpr());
- checkMatchingDimensions(getFirstExpr(), getThirdExpr());
- }
-
- // output is a scalar
- output.setDataType(DataType.SCALAR);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- break;
-
- case QUANTILE:
- /*
- * q = quantile(V1,0.5) computes median in V1
- * or Q = quantile(V1,P) computes the vector of quantiles as specified by P
- * or qw = quantile(V1,W,0.5) computes median when weights (W) are given
- * or QW = quantile(V1,W,P) computes the vector of quantiles as specified by P, when weights (W) are given
- */
- if(getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
-
- // first parameter must always be a 1D matrix
- check1DMatrixParam(getFirstExpr());
-
- // check for matching dimensions for other matrix parameters
- if (getThirdExpr() != null) {
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // set the properties for _output expression
- // output dimensions = dimensions of second, if third is null
- // = dimensions of the third, otherwise.
-
- if (getThirdExpr() != null) {
- output.setDimensions(getThirdExpr().getOutput().getDim1(), getThirdExpr().getOutput()
- .getDim2());
- output.setBlockDimensions(getThirdExpr().getOutput().getRowsInBlock(),
- getThirdExpr().getOutput().getColumnsInBlock());
- output.setDataType(getThirdExpr().getOutput().getDataType());
- } else {
- output.setDimensions(getSecondExpr().getOutput().getDim1(), getSecondExpr().getOutput()
- .getDim2());
- output.setBlockDimensions(getSecondExpr().getOutput().getRowsInBlock(),
- getSecondExpr().getOutput().getColumnsInBlock());
- output.setDataType(getSecondExpr().getOutput().getDataType());
- }
- break;
-
- case INTERQUANTILE:
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
- checkMatrixParam(getFirstExpr());
- if (getThirdExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensionsQuantile();
- }
-
- if ((getThirdExpr() == null && getSecondExpr().getOutput().getDataType() != DataType.SCALAR)
- && (getThirdExpr() != null && getThirdExpr().getOutput().getDataType() != DataType.SCALAR)) {
-
- raiseValidateError("Invalid parameters to "+ this.getOpCode(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- output.setValueType(id.getValueType());
- // output dimensions are unknown
- output.setDimensions(-1, -1);
- output.setBlockDimensions(-1,-1);
- output.setDataType(DataType.MATRIX);
- break;
-
- case IQM:
- /*
- * Usage: iqm = InterQuartileMean(A,W); iqm = InterQuartileMean(A);
- */
- if (getSecondExpr() != null){
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- checkMatrixParam(getFirstExpr());
-
- if (getSecondExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // Output is a scalar
- output.setValueType(id.getValueType());
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- output.setDataType(DataType.SCALAR);
-
- break;
-
- case MEDIAN:
- if (getSecondExpr() != null){
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- checkMatrixParam(getFirstExpr());
-
- if (getSecondExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // Output is a scalar
- output.setValueType(id.getValueType());
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- output.setDataType(DataType.SCALAR);
-
- break;
-
- case SAMPLE:
- {
- Expression[] in = getAllExpr();
-
- for(Expression e : in)
- checkScalarParam(e);
-
- if (in[0].getOutput().getValueType() != ValueType.DOUBLE && in[0].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("First argument to sample() must be a number.");
- if (in[1].getOutput().getValueType() != ValueType.DOUBLE && in[1].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("Second argument to sample() must be a number.");
-
- boolean check = false;
- if ( isConstant(in[0]) && isConstant(in[1]) )
- {
- long range = ((ConstIdentifier)in[0]).getLongValue();
- long size = ((ConstIdentifier)in[1]).getLongValue();
- if ( range < size )
- check = true;
- }
-
- if(in.length == 4 )
- {
- checkNumParameters(4);
- if (in[3].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("Fourth arugment, seed, to sample() must be an integer value.");
- if (in[2].getOutput().getValueType() != ValueType.BOOLEAN )
- throw new LanguageException("Third arugment to sample() must either denote replacement policy (boolean) or seed (integer).");
- }
- else if(in.length == 3)
- {
- checkNumParameters(3);
- if (in[2].getOutput().getValueType() != ValueType.BOOLEAN
- && in[2].getOutput().getValueType() != ValueType.INT )
- throw new LanguageException("Third arugment to sample() must either denote replacement policy (boolean) or seed (integer).");
- }
-
- if ( check && in.length >= 3
- && isConstant(in[2])
- && in[2].getOutput().getValueType() == ValueType.BOOLEAN
- && !((BooleanIdentifier)in[2]).getValue() )
- throw new LanguageException("Sample (size=" + ((ConstIdentifier)in[0]).getLongValue()
- + ") larger than population (size=" + ((ConstIdentifier)in[1]).getLongValue()
- + ") can only be generated with replacement.");
-
- // Output is a column vector
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
-
- if ( isConstant(in[1]) )
- output.setDimensions(((ConstIdentifier)in[1]).getLongValue(), 1);
- else
- output.setDimensions(-1, 1);
- setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
-
- break;
- }
- case SEQ:
-
- //basic parameter validation
- checkScalarParam(getFirstExpr());
- checkScalarParam(getSecondExpr());
- if ( getThirdExpr() != null ) {
- checkNumParameters(3);
- checkScalarParam(getThirdExpr());
- }
- else
- checkNumParameters(2);
-
- // constant propagation (from, to, incr)
- if( getFirstExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getFirstExpr()).getName()) )
- _args[0] = constVars.get(((DataIdentifier)getFirstExpr()).getName());
- if( getSecondExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getSecondExpr()).getName()) )
- _args[1] = constVars.get(((DataIdentifier)getSecondExpr()).getName());
- if( getThirdExpr()!=null && getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) )
- _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName());
-
- // check if dimensions can be inferred
- long dim1=-1, dim2=1;
- if ( isConstant(getFirstExpr()) && isConstant(getSecondExpr()) && (getThirdExpr() != null ? isConstant(getThirdExpr()) : true) ) {
- double from, to, incr;
- boolean neg;
- try {
- from = getDoubleValue(getFirstExpr());
- to = getDoubleValue(getSecondExpr());
-
- // Setup the value of increment
- // default value: 1 if from <= to; -1 if from > to
- neg = (from > to);
- if(getThirdExpr() == null) {
- expandArguments();
- _args[2] = new DoubleIdentifier((neg? -1.0 : 1.0),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- }
- incr = getDoubleValue(getThirdExpr());
-
- }
- catch (LanguageException e) {
- throw new LanguageException("Arguments for seq() must be numeric.");
- }
-
- if (neg != (incr < 0))
- throw new LanguageException("Wrong sign for the increment in a call to seq()");
-
- // Both end points of the range must included i.e., [from,to] both inclusive.
- // Note that, "to" is included only if (to-from) is perfectly divisible by incr
- // For example, seq(0,1,0.5) produces (0.0 0.5 1.0) whereas seq(0,1,0.6) produces only (0.0 0.6) but not (0.0 0.6 1.0)
- dim1 = 1 + (long)Math.floor((to-from)/incr);
- //System.out.println("seq("+from+","+to+","+incr+") -> dims("+dim1+","+dim2+")");
- }
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(dim1, dim2);
- output.setBlockDimensions(0, 0);
- break;
-
- case SOLVE:
- checkNumParameters(2);
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
-
- if ( getSecondExpr().getOutput().dimsKnown() && !is1DMatrix(getSecondExpr()) )
- raiseValidateError("Second input to solve() must be a vector", conditional);
-
- if ( getFirstExpr().getOutput().dimsKnown() && getSecondExpr().getOutput().dimsKnown() &&
- getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() )
- raiseValidateError("Dimension mismatch in a call to solve()", conditional);
-
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(getFirstExpr().getOutput().getDim2(), 1);
- output.setBlockDimensions(0, 0);
- break;
-
- case INVERSE:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
-
- Identifier in = getFirstExpr().getOutput();
- if(in.dimsKnown() && in.getDim1() != in.getDim2())
- raiseValidateError("Input to inv() must be square matrix -- given: a " + in.getDim1() + "x" + in.getDim2() + " matrix.", conditional);
-
- output.setDimensions(in.getDim1(), in.getDim2());
- output.setBlockDimensions(in.getRowsInBlock(), in.getColumnsInBlock());
- break;
-
- case OUTER:
- Identifier id2 = this.getSecondExpr().getOutput();
-
- //check input types and characteristics
- checkNumParameters(3);
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- checkScalarParam(getThirdExpr());
- checkValueTypeParam(getThirdExpr(), ValueType.STRING);
- if( id.getDim2() > 1 || id2.getDim1()>1 ) {
- raiseValidateError("Outer vector operations require a common dimension of one: " +
- id.getDim1()+"x"+id.getDim2()+" o "+id2.getDim1()+"x"+id2.getDim2()+".", false);
- }
-
- //set output characteristics
- output.setDataType(id.getDataType());
- output.setDimensions(id.getDim1(), id2.getDim2());
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- break;
-
- default:
- if (this.isMathFunction()) {
- // datatype and dimensions are same as this.getExpr()
- if (this.getOpCode() == BuiltinFunctionOp.ABS) {
- output.setValueType(getFirstExpr().getOutput().getValueType());
- } else {
- output.setValueType(ValueType.DOUBLE);
- }
- checkMathFunctionParam();
- output.setDataType(id.getDataType());
- output.setDimensions(id.getDim1(), id.getDim2());
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- } else{
- // always unconditional (because unsupported operation)
- raiseValidateError("Unsupported function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- return;
- }
-
- private void expandArguments() {
-
- if ( _args == null ) {
- _args = new Expression[1];
- return;
- }
- Expression [] temp = _args.clone();
- _args = new Expression[_args.length + 1];
- System.arraycopy(temp, 0, _args, 0, temp.length);
- }
-
- @Override
- public boolean multipleReturns() {
- switch(_opcode) {
- case QR:
- case LU:
- case EIGEN:
- return true;
- default:
- return false;
- }
- }
-
- /**
- *
- * @param expr
- * @return
- */
- private boolean isConstant(Expression expr) {
- return ( expr != null && expr instanceof ConstIdentifier );
- }
-
- /**
- *
- * @param expr
- * @return
- * @throws LanguageException
- */
- private double getDoubleValue(Expression expr)
- throws LanguageException
- {
- if ( expr instanceof DoubleIdentifier )
- return ((DoubleIdentifier)expr).getValue();
- else if ( expr instanceof IntIdentifier)
- return ((IntIdentifier)expr).getValue();
- else
- throw new LanguageException("Expecting a numeric value.");
- }
-
- private boolean isMathFunction() {
- switch (this.getOpCode()) {
- case COS:
- case SIN:
- case TAN:
- case ACOS:
- case ASIN:
- case ATAN:
- case SQRT:
- case ABS:
- case LOG:
- case EXP:
- case ROUND:
- case CEIL:
- case FLOOR:
- case MEDIAN:
- return true;
- default:
- return false;
- }
- }
-
- private void checkMathFunctionParam() throws LanguageException {
- switch (this.getOpCode()) {
- case COS:
- case SIN:
- case TAN:
- case ACOS:
- case ASIN:
- case ATAN:
- case SQRT:
- case ABS:
- case EXP:
- case ROUND:
- case CEIL:
- case FLOOR:
- case MEDIAN:
- checkNumParameters(1);
- break;
- case LOG:
- if (getSecondExpr() != null) {
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- break;
- default:
- //always unconditional
- raiseValidateError("Unknown math function "+ this.getOpCode(), false);
- }
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder(_opcode.toString() + "(" + _args[0].toString());
- for(int i=1; i < _args.length; i++) {
- sb.append(",");
- sb.append(_args[i].toString());
- }
- sb.append(")");
- return sb.toString();
- }
-
- @Override
- // third part of expression IS NOT a variable -- it is the OP to be applied
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- for(int i=0; i<_args.length; i++) {
- result.addVariables(_args[i].variablesRead());
- }
-
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- // result.addVariables(_first.variablesUpdated());
- return result;
- }
-
- /**
- *
- * @param count
- * @throws LanguageException
- */
- protected void checkNumParameters(int count) //always unconditional
- throws LanguageException
- {
- if (getFirstExpr() == null){
- raiseValidateError("Missing parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- if (((count == 1) && (getSecondExpr()!= null || getThirdExpr() != null)) ||
- ((count == 2) && (getThirdExpr() != null))){
- raiseValidateError("Invalid number of parameters for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if (((count == 2) && (getSecondExpr() == null)) ||
- ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))){
- raiseValidateError( "Missing parameter for function "+this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- protected void checkMatrixParam(Expression e) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getDataType() != DataType.MATRIX) {
- raiseValidateError("Expecting matrix parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- private void checkScalarParam(Expression e) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getDataType() != DataType.SCALAR)
- {
- raiseValidateError("Expecting scalar parameter for function " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @param vt
- * @throws LanguageException
- */
- private void checkValueTypeParam(Expression e, ValueType vt) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getValueType() != vt) {
- raiseValidateError("Expecting parameter of different value type " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- private boolean is1DMatrix(Expression e) {
- return (e.getOutput().getDim1() == 1 || e.getOutput().getDim2() == 1 );
- }
-
- private boolean dimsKnown(Expression e) {
- return (e.getOutput().getDim1() != -1 && e.getOutput().getDim2() != -1);
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- private void check1DMatrixParam(Expression e) //always unconditional
- throws LanguageException
- {
- checkMatrixParam(e);
-
- // throw an exception, when e's output is NOT a one-dimensional matrix
- // the check must be performed only when the dimensions are known at compilation time
- if ( dimsKnown(e) && !is1DMatrix(e)) {
- raiseValidateError("Expecting one-dimensional matrix parameter for function "
- + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param expr1
- * @param expr2
- * @throws LanguageException
- */
- private void checkMatchingDimensions(Expression expr1, Expression expr2)
- throws LanguageException
- {
- checkMatchingDimensions(expr1, expr2, false);
- }
-
- /**
- *
- * @param expr1
- * @param expr2
- * @throws LanguageException
- */
- private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV)
- throws LanguageException
- {
- if (expr1 != null && expr2 != null) {
-
- // if any matrix has unknown dimensions, simply return
- if( expr1.getOutput().getDim1() == -1 || expr2.getOutput().getDim1() == -1
- ||expr1.getOutput().getDim2() == -1 || expr2.getOutput().getDim2() == -1 )
- {
- return;
- }
- else if( (!allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1())
- || (allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1() && expr2.getOutput().getDim1() != 1)
- || (!allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2())
- || (allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2() && expr2.getOutput().getDim2() != 1) )
- {
- raiseValidateError("Mismatch in matrix dimensions of parameters for function "
- + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- /**
- *
- * @throws LanguageException
- */
- private void checkMatchingDimensionsQuantile()
- throws LanguageException
- {
- if (getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1()) {
- raiseValidateError("Mismatch in matrix dimensions for "
- + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- public static BuiltinFunctionExpression getBuiltinFunctionExpression(
- String functionName, ArrayList<ParameterExpression> paramExprsPassed,
- String filename, int blp, int bcp, int elp, int ecp) {
-
- if (functionName == null || paramExprsPassed == null)
- return null;
-
- // check if the function name is built-in function
- // (assign built-in function op if function is built-in
- Expression.BuiltinFunctionOp bifop = null;
-
- if (functionName.equals("avg"))
- bifop = Expression.BuiltinFunctionOp.MEAN;
- else if (functionName.equals("cos"))
- bifop = Expression.BuiltinFunctionOp.COS;
- else if (functionName.equals("sin"))
- bifop = Expression.BuiltinFunctionOp.SIN;
- else if (functionName.equals("tan"))
- bifop = Expression.BuiltinFunctionOp.TAN;
- else if (functionName.equals("acos"))
- bifop = Expression.BuiltinFunctionOp.ACOS;
- else if (functionName.equals("asin"))
- bifop = Expression.BuiltinFunctionOp.ASIN;
- else if (functionName.equals("atan"))
- bifop = Expression.BuiltinFunctionOp.ATAN;
- else if (functionName.equals("diag"))
- bifop = Expression.BuiltinFunctionOp.DIAG;
- else if (functionName.equals("exp"))
- bifop = Expression.BuiltinFunctionOp.EXP;
- else if (functionName.equals("abs"))
- bifop = Expression.BuiltinFunctionOp.ABS;
- else if (functionName.equals("min"))
- bifop = Expression.BuiltinFunctionOp.MIN;
- else if (functionName.equals("max"))
- bifop = Expression.BuiltinFunctionOp.MAX;
- //NOTE: pmin and pmax are just kept for compatibility to R
- // min and max is capable of handling all unary and binary
- // operations (in contrast to R)
- else if (functionName.equals("pmin"))
- bifop = Expression.BuiltinFunctionOp.MIN;
- else if (functionName.equals("pmax"))
- bifop = Expression.BuiltinFunctionOp.MAX;
- else if (functionName.equals("ppred"))
- bifop = Expression.BuiltinFunctionOp.PPRED;
- else if (functionName.equals("log"))
- bifop = Expression.BuiltinFunctionOp.LOG;
- else if (functionName.equals("length"))
- bifop = Expression.BuiltinFunctionOp.LENGTH;
- else if (functionName.equals("ncol"))
- bifop = Expression.BuiltinFunctionOp.NCOL;
- else if (functionName.equals("nrow"))
- bifop = Expression.BuiltinFunctionOp.NROW;
- else if (functionName.equals("sqrt"))
- bifop = Expression.BuiltinFunctionOp.SQRT;
- else if (functionName.equals("sum"))
- bifop = Expression.BuiltinFunctionOp.SUM;
- else if (functionName.equals("mean"))
- bifop = Expression.BuiltinFunctionOp.MEAN;
- else if (functionName.equals("trace"))
- bifop = Expression.BuiltinFunctionOp.TRACE;
- else if (functionName.equals("t"))
- bifop = Expression.BuiltinFunctionOp.TRANS;
- else if (functionName.equals("cbind") || functionName.equals("append"))
- bifop = Expression.BuiltinFunctionOp.CBIND;
- else if (functionName.equals("rbind"))
- bifop = Expression.BuiltinFunctionOp.RBIND;
- else if (functionName.equals("range"))
- bifop = Expression.BuiltinFunctionOp.RANGE;
- else if (functionName.equals("prod"))
- bifop = Expression.BuiltinFunctionOp.PROD;
- else if (functionName.equals("rowSums"))
- bifop = Expression.BuiltinFunctionOp.ROWSUM;
- else if (functionName.equals("colSums"))
- bifop = Expression.BuiltinFunctionOp.COLSUM;
- else if (functionName.equals("rowMins"))
- bifop = Expression.BuiltinFunctionOp.ROWMIN;
- else if (functionName.equals("colMins"))
- bifop = Expression.BuiltinFunctionOp.COLMIN;
- else if (functionName.equals("rowMaxs"))
- bifop = Expression.BuiltinFunctionOp.ROWMAX;
- else if (functionName.equals("rowIndexMax"))
- bifop = Expression.BuiltinFunctionOp.ROWINDEXMAX;
- else if (functionName.equals("rowIndexMin"))
- bifop = Expression.BuiltinFunctionOp.ROWINDEXMIN;
- else if (functionName.equals("colMaxs"))
- bifop = Expression.BuiltinFunctionOp.COLMAX;
- else if (functionName.equals("rowMeans"))
- bifop = Expression.BuiltinFunctionOp.ROWMEAN;
- else if (functionName.equals("colMeans"))
- bifop = Expression.BuiltinFunctionOp.COLMEAN;
- else if (functionName.equals("cummax"))
- bifop = Expression.BuiltinFunctionOp.CUMMAX;
- else if (functionName.equals("cummin"))
- bifop = Expression.BuiltinFunctionOp.CUMMIN;
- else if (functionName.equals("cumprod"))
- bifop = Expression.BuiltinFunctionOp.CUMPROD;
- else if (functionName.equals("cumsum"))
- bifop = Expression.BuiltinFunctionOp.CUMSUM;
- //'castAsScalar' for backwards compatibility
- else if (functionName.equals("as.scalar") || functionName.equals("castAsScalar"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_SCALAR;
- else if (functionName.equals("as.matrix"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_MATRIX;
- else if (functionName.equals("as.double"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_DOUBLE;
- else if (functionName.equals("as.integer"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_INT;
- else if (functionName.equals("as.logical")) //alternative: as.boolean
- bifop = Expression.BuiltinFunctionOp.CAST_AS_BOOLEAN;
- else if (functionName.equals("quantile"))
- bifop= Expression.BuiltinFunctionOp.QUANTILE;
- else if (functionName.equals("interQuantile"))
- bifop= Expression.BuiltinFunctionOp.INTERQUANTILE;
- else if (functionName.equals("interQuartileMean"))
- bifop= Expression.BuiltinFunctionOp.IQM;
- //'ctable' for backwards compatibility
- else if (functionName.equals("table") || functionName.equals("ctable"))
- bifop = Expression.BuiltinFunctionOp.TABLE;
- else if (functionName.equals("round"))
- bifop = Expression.BuiltinFunctionOp.ROUND;
- //'centralMoment' for backwards compatibility
- else if (functionName.equals("moment") || functionName.equals("centralMoment"))
- bifop = Expression.BuiltinFunctionOp.MOMENT;
- else if (functionName.equals("cov"))
- bifop = Expression.BuiltinFunctionOp.COV;
- else if (functionName.equals("seq"))
- bifop = Expression.BuiltinFunctionOp.SEQ;
- else if (functionName.equals("qr"))
- bifop = Expression.BuiltinFunctionOp.QR;
- else if (functionName.equals("lu"))
- bifop = Expression.BuiltinFunctionOp.LU;
- else if (functionName.equals("eigen"))
- bifop = Expression.BuiltinFunctionOp.EIGEN;
- else if (functionName.equals("solve"))
- bifop = Expression.BuiltinFunctionOp.SOLVE;
- else if (functionName.equals("ceil"))
- bifop = Expression.BuiltinFunctionOp.CEIL;
- else if (functionName.equals("floor"))
- bifop = Expression.BuiltinFunctionOp.FLOOR;
- else if (functionName.equals("median"))
- bifop = Expression.BuiltinFunctionOp.MEDIAN;
- else if (functionName.equals("inv"))
- bifop = Expression.BuiltinFunctionOp.INVERSE;
- else if (functionName.equals("sample"))
- bifop = Expression.BuiltinFunctionOp.SAMPLE;
- else if ( functionName.equals("outer") )
- bifop = Expression.BuiltinFunctionOp.OUTER;
- else
- return null;
-
- BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(bifop, paramExprsPassed,
- filename, blp, bcp, elp, ecp);
-
- return retVal;
- } // end method getBuiltinFunctionExpression
-
- /**
- *
- * @param vt
- * @return
- * @throws LanguageException
- */
- public static BuiltinFunctionOp getValueTypeCastOperator( ValueType vt )
- throws LanguageException
- {
- switch( vt )
- {
- case DOUBLE:
- return BuiltinFunctionOp.CAST_AS_DOUBLE;
- case INT:
- return BuiltinFunctionOp.CAST_AS_INT;
- case BOOLEAN:
- return BuiltinFunctionOp.CAST_AS_BOOLEAN;
- default:
- throw new LanguageException("No cast for value type "+vt);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java b/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
deleted file mode 100644
index 0b76aaa..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-public class ConditionalPredicate
-{
-
-
- Expression _expr;
-
-
- public ConditionalPredicate(Expression expr){
- _expr = expr;
- }
-
- public Expression getPredicate(){
- return _expr;
- }
- public void setPredicate(Expression expr){
- _expr = expr;
- }
-
- public String toString(){
- return _expr.toString();
- }
-
-
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_expr.variablesRead());
- return result;
- }
-
-
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_expr.variablesUpdated());
- return result;
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for expressions
- ///////////////////////////////////////////////////////////////////////////
- private String _filename;
- private int _beginLine, _beginColumn;
- private int _endLine, _endColumn;
-
- public void setFilename(String fname) { _filename = fname; }
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(String fname, int blp, int bcp, int elp, int ecp){
- _filename = fname;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public String getFilename() { return _filename; }
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
deleted file mode 100644
index 947484e..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-public abstract class ConstIdentifier extends Identifier
-{
-
- public ConstIdentifier(){
- super();
- }
-
- public abstract long getLongValue()
- throws LanguageException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java b/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
deleted file mode 100644
index 8baad0c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-public class DMLParseException extends ParseException
-{
-
-
- /**
- * The version identifier for this Serializable class.
- * Increment only if the <i>serialized</i> form of the
- * class changes.
- */
- private static final long serialVersionUID = 1L;
-
- private String _filename;
-
- private ArrayList<DMLParseException> _exceptionList;
-
- public ArrayList<DMLParseException> getExceptionList(){
- return _exceptionList;
- }
-
- public DMLParseException(String fname){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- }
-
- public DMLParseException(String fname, String msg){
- super(msg);
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- _exceptionList.add(this);
- }
-
- public DMLParseException(String fname, Exception e){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- String newMsg = e.getMessage();
- if (e instanceof ParseException && !(e instanceof DMLParseException)){
- ParseException parseEx = (ParseException)e;
- int beginLine = -1, beginColumn = -1;
- String errorToken = null;
- if (parseEx.currentToken != null){
- beginLine = parseEx.currentToken.beginLine;
- beginColumn = parseEx.currentToken.beginColumn;
- errorToken = parseEx.currentToken.image;
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error around token \"" + errorToken + "\"";
- } else {
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error with unspecified token";
- }
- }
-
- _exceptionList.add(new DMLParseException(_filename, newMsg));
- }
-
- public int size(){
- return _exceptionList.size();
- }
-
- public void add(Exception e){
- if (e instanceof DMLParseException)
- _exceptionList.addAll(((DMLParseException)e).getExceptionList());
- else
- _exceptionList.add(new DMLParseException(this._filename, e));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java b/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
deleted file mode 100644
index 104c231..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
+++ /dev/null
@@ -1,783 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-
-
-
-public class DMLProgram
-{
-
- private ArrayList<StatementBlock> _blocks;
- private HashMap<String, FunctionStatementBlock> _functionBlocks;
- private HashMap<String,DMLProgram> _namespaces;
- public static String DEFAULT_NAMESPACE = ".defaultNS";
- public static String INTERNAL_NAMESPACE = "_internal"; // used for multi-return builtin functions
- private static final Log LOG = LogFactory.getLog(DMLProgram.class.getName());
-
- public DMLProgram(){
- _blocks = new ArrayList<StatementBlock>();
- _functionBlocks = new HashMap<String,FunctionStatementBlock>();
- _namespaces = new HashMap<String,DMLProgram>();
- _namespaces.put(DMLProgram.DEFAULT_NAMESPACE,this);
- }
-
- public HashMap<String,DMLProgram> getNamespaces(){
- return _namespaces;
- }
-
- public void addStatementBlock(StatementBlock b, int pos) {
- _blocks.add(pos,b) ;
- }
-
- public void addStatementBlock(StatementBlock b){
- _blocks.add(b);
- }
-
- public int getNumStatementBlocks(){
- return _blocks.size();
- }
-
- public FunctionStatementBlock getFunctionStatementBlock(String namespaceKey, String functionName) {
- DMLProgram namespaceProgram = this.getNamespaces().get(namespaceKey);
- if (namespaceProgram == null)
- return null;
-
- // for the namespace DMLProgram, get the specified function (if exists) in its current namespace
- FunctionStatementBlock retVal = namespaceProgram._functionBlocks.get(functionName);
- return retVal;
- }
-
- public HashMap<String, FunctionStatementBlock> getFunctionStatementBlocks(String namespaceKey) throws LanguageException{
- DMLProgram namespaceProgram = this.getNamespaces().get(namespaceKey);
- if (namespaceProgram == null){
- LOG.error("ERROR: namespace " + namespaceKey + " is underfined");
- throw new LanguageException("ERROR: namespace " + namespaceKey + " is underfined");
- }
- // for the namespace DMLProgram, get the functions in its current namespace
- return namespaceProgram._functionBlocks;
- }
-
- public ArrayList<FunctionStatementBlock> getFunctionStatementBlocks()
- throws LanguageException
- {
- ArrayList<FunctionStatementBlock> ret = new ArrayList<FunctionStatementBlock>();
-
- for( DMLProgram nsProg : _namespaces.values() )
- ret.addAll(nsProg._functionBlocks.values());
-
- return ret;
- }
-
- public void addFunctionStatementBlock( String namespace, String fname, FunctionStatementBlock fsb )
- throws LanguageException
- {
- DMLProgram namespaceProgram = this.getNamespaces().get(namespace);
- if (namespaceProgram == null)
- throw new LanguageException( "Namespace does not exist." );
-
- namespaceProgram._functionBlocks.put(fname, fsb);
- }
-
- public ArrayList<StatementBlock> getStatementBlocks(){
- return _blocks;
- }
-
- public void setStatementBlocks(ArrayList<StatementBlock> passed){
- _blocks = passed;
- }
-
- public StatementBlock getStatementBlock(int i){
- return _blocks.get(i);
- }
-
- public void setStatementBlock(int i, StatementBlock sb) {
- _blocks.set(i, sb);
- }
-
- public void mergeStatementBlocks(){
- _blocks = StatementBlock.mergeStatementBlocks(_blocks);
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
-
- // for each namespace, display all functions
- for (String namespaceKey : this.getNamespaces().keySet()){
-
- sb.append("NAMESPACE = " + namespaceKey + "\n");
- DMLProgram namespaceProg = this.getNamespaces().get(namespaceKey);
-
-
- sb.append("FUNCTIONS = ");
-
- for (FunctionStatementBlock fsb : namespaceProg._functionBlocks.values()){
- sb.append(fsb);
- sb.append(", ");
- }
- sb.append("\n");
- sb.append("********************************** \n");
-
- }
-
- sb.append("******** MAIN SCRIPT BODY ******** \n");
- for (StatementBlock b : _blocks){
- sb.append(b);
- sb.append("\n");
- }
- sb.append("********************************** \n");
- return sb.toString();
- }
-
-
- public Program getRuntimeProgram(DMLConfig config) throws IOException, LanguageException, DMLRuntimeException, LopsException, DMLUnsupportedOperationException {
-
- // constructor resets the set of registered functions
- Program rtprog = new Program();
-
- // for all namespaces, translate function statement blocks into function program blocks
- for (String namespace : _namespaces.keySet()){
-
- for (String fname : getFunctionStatementBlocks(namespace).keySet()){
- // add program block to program
- FunctionStatementBlock fsb = getFunctionStatementBlocks(namespace).get(fname);
- FunctionProgramBlock rtpb = (FunctionProgramBlock)createRuntimeProgramBlock(rtprog, fsb, config);
- rtprog.addFunctionProgramBlock(namespace, fname, rtpb);
- rtpb.setRecompileOnce( fsb.isRecompileOnce() );
- }
- }
-
- // for each top-level block
- for (StatementBlock sb : _blocks) {
-
- // add program block to program
- ProgramBlock rtpb = createRuntimeProgramBlock(rtprog, sb, config);
- rtprog.addProgramBlock(rtpb);
- }
-
-
- return rtprog ;
- }
-
- /**
- *
- * @param prog
- * @param sb
- * @param config
- * @return
- * @throws IOException
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config)
- throws IOException, LopsException, DMLRuntimeException, DMLUnsupportedOperationException
- {
- Dag<Lop> dag = null;
- Dag<Lop> pred_dag = null;
-
- ArrayList<Instruction> instruct;
- ArrayList<Instruction> pred_instruct = null;
-
- ProgramBlock retPB = null;
-
- // process While Statement - add runtime program blocks to program
- if (sb instanceof WhileStatementBlock){
-
- // create DAG for loop predicates
- pred_dag = new Dag<Lop>();
- ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
-
- // create instructions for loop predicates
- pred_instruct = new ArrayList<Instruction>();
- ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
- for (Instruction i : pInst ) {
- pred_instruct.add(i);
- }
-
- // create while program block
- WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
-
- if (rtpb.getPredicateResultVar() == null) {
- // e.g case : WHILE(continue)
- if ( ((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
-
- }
- }
- //// process the body of the while statement block ////
-
- WhileStatementBlock wsb = (WhileStatementBlock)sb;
- if (wsb.getNumStatements() > 1){
- LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
- throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
- }
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- for (StatementBlock sblock : wstmt.getBody()){
-
- // process the body
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (wsb.getLops() != null && !wsb.getLops().isEmpty() ){
- LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
- throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
- }
-
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process If Statement - add runtime program blocks to program
- else if (sb instanceof IfStatementBlock){
-
- // create DAG for loop predicates
- pred_dag = new Dag<Lop>();
- ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
-
- // create instructions for loop predicates
- pred_instruct = new ArrayList<Instruction>();
- ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
- for (Instruction i : pInst ) {
- pred_instruct.add(i);
- }
-
- // create if program block
- IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
-
- if (rtpb.getPredicateResultVar() == null ) {
- // e.g case : If(continue)
- if ( ((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- }
- }
-
- // process the body of the if statement block
- IfStatementBlock isb = (IfStatementBlock)sb;
- if (isb.getNumStatements() > 1){
- LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
- throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
- }
- IfStatement istmt = (IfStatement)isb.getStatement(0);
-
- // process the if body
- for (StatementBlock sblock : istmt.getIfBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlockIfBody(childBlock);
- }
-
- // process the else body
- for (StatementBlock sblock : istmt.getElseBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlockElseBody(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (isb.getLops() != null && !isb.getLops().isEmpty() ){
- LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
- throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
- }
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process For Statement - add runtime program blocks to program
- // NOTE: applies to ForStatementBlock and ParForStatementBlock
- else if (sb instanceof ForStatementBlock)
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
-
- // create DAGs for loop predicates
- Dag<Lop> fromDag = new Dag<Lop>();
- Dag<Lop> toDag = new Dag<Lop>();
- Dag<Lop> incrementDag = new Dag<Lop>();
- if( fsb.getFromHops()!=null )
- fsb.getFromLops().addToDag(fromDag);
- if( fsb.getToHops()!=null )
- fsb.getToLops().addToDag(toDag);
- if( fsb.getIncrementHops()!=null )
- fsb.getIncrementLops().addToDag(incrementDag);
-
- // create instructions for loop predicates
- ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
- ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
- ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
-
- // create for program block
- String sbName = null;
- ForProgramBlock rtpb = null;
- IterablePredicate iterPred = fsb.getIterPredicate();
- String [] iterPredData= IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(),
- fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
-
- if( sb instanceof ParForStatementBlock )
- {
- sbName = "ParForStatementBlock";
- rtpb = new ParForProgramBlock(prog, iterPredData,iterPred.getParForParams());
- ParForProgramBlock pfrtpb = (ParForProgramBlock)rtpb;
- pfrtpb.setResultVariables( ((ParForStatementBlock)sb).getResultVariables() );
- pfrtpb.setStatementBlock((ParForStatementBlock)sb); //used for optimization and creating unscoped variables
- }
- else //ForStatementBlock
- {
- sbName = "ForStatementBlock";
- rtpb = new ForProgramBlock(prog, iterPredData);
- }
-
- rtpb.setFromInstructions( fromInstructions );
- rtpb.setToInstructions( toInstructions );
- rtpb.setIncrementInstructions( incrementInstructions );
-
- rtpb.setIterablePredicateVars( iterPredData );
-
- // process the body of the for statement block
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement" );
- throw new LopsException(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement" );
- }
- ForStatement fs = (ForStatement)fsb.getStatement(0);
- for (StatementBlock sblock : fs.getBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (fsb.getLops() != null && !fsb.getLops().isEmpty()){
- LOG.error(fsb.printBlockErrorLocation() + sbName + " should have no Lops" );
- throw new LopsException(fsb.printBlockErrorLocation() + sbName + " should have no Lops" );
- }
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process function statement block - add runtime program blocks to program
- else if (sb instanceof FunctionStatementBlock){
-
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
- throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
- }
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- FunctionProgramBlock rtpb = null;
-
- if (fstmt instanceof ExternalFunctionStatement) {
- // create external function program block
-
- String execType = ((ExternalFunctionStatement) fstmt)
- .getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
- boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
-
- String scratchSpaceLoc = null;
- try {
- scratchSpaceLoc = config.getTextValue(DMLConfig.SCRATCH_SPACE);
- } catch (Exception e){
- LOG.error(fsb.printBlockErrorLocation() + "could not retrieve parameter " + DMLConfig.SCRATCH_SPACE + " from DMLConfig");
- }
- StringBuilder buff = new StringBuilder();
- buff.append(scratchSpaceLoc);
- buff.append(Lop.FILE_SEPARATOR);
- buff.append(Lop.PROCESS_PREFIX);
- buff.append(DMLScript.getUUID());
- buff.append(Lop.FILE_SEPARATOR);
- buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
- buff.append(Lop.FILE_SEPARATOR);
- buff.append("PackageSupport");
- buff.append(Lop.FILE_SEPARATOR);
- String basedir = buff.toString();
-
- if( isCP )
- {
-
- rtpb = new ExternalFunctionProgramBlockCP(prog,
- fstmt.getInputParams(), fstmt.getOutputParams(),
- ((ExternalFunctionStatement) fstmt).getOtherParams(),
- basedir );
- }
- else
- {
- rtpb = new ExternalFunctionProgramBlock(prog,
- fstmt.getInputParams(), fstmt.getOutputParams(),
- ((ExternalFunctionStatement) fstmt).getOtherParams(),
- basedir);
- }
-
- if (!fstmt.getBody().isEmpty()){
- LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
- throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
- }
- }
- else
- {
- // create function program block
- rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
-
- // process the function statement body
- for (StatementBlock sblock : fstmt.getBody()){
- // process the body
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (fsb.getLops() != null && !fsb.getLops().isEmpty()){
- LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
- throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
- }
-
- retPB = rtpb;
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
- else {
-
- // handle general case
- ProgramBlock rtpb = new ProgramBlock(prog);
-
- // DAGs for Lops
- dag = new Dag<Lop>();
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (sb.getLops() != null && !sb.getLops().isEmpty()){
-
- for (Lop l : sb.getLops()) {
- l.addToDag(dag);
- }
-
- // Instructions for Lobs DAGs
- instruct = dag.getJobs(sb, config);
- rtpb.addInstructions(instruct);
- }
-
- /*// TODO: check with Doug
- // add instruction for a function call
- if (sb.getFunctionCallInst() != null){
- rtpb.addInstruction(sb.getFunctionCallInst());
- }*/
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- return retPB;
- }
-
- /**
- * Post processing of each created program block in order to adhere to livein/liveout
- * (currently needed for cleanup (especially for caching) of intermediate results if the last datasink
- * is an external function because instructions of external functions are created outside hops/lops,
- * e.g., X=..., Y=fun(X) and X is not used afterwards )
- *
- * NOTES:
- * (1) Rule1: checking livein and liveout is sufficient because the last external function is in its own
- * programblock anyway.
- * (2) as we cannot efficiently distinguish if the problematic var is created by an external function
- * or some other instruction, we generate RMVAR instructions although for vars created by non-CP
- * external functions RMFILEVAR instructions are required. However, all remaining files in scratch_space
- * are cleaned after execution anyway.
- * (3) As an alternative to doing rule 2, we could also check for existing objects in createvar and function invocation
- * (or generic at program block level) and remove objects of previous iterations accordingly (but objects of last iteration
- * would still require seperate cleanup).
- *
- * TODO: MB: external function invocations should become hops/lops as well (see instruction gen in DMLTranslator),
- * (currently not possible at Hops/Lops level due the requirement of multiple outputs for functions)
- * TODO: MB: we should in general always leverage livein/liveout during hops/lops generation.
- * TODO: MB: verify and correct can be removed once everything is integrated in hops/lops generation
- *
- * @param in
- * @param out
- * @param pb
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unused")
- private ProgramBlock verifyAndCorrectProgramBlock(VariableSet in, VariableSet out, VariableSet kill, ProgramBlock pb)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //RULE 1: if in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- //(currently required for specific cases of external functions)
- for( String varName : in.getVariableNames() )
- if( !out.containsVariable(varName) )
- {
- DataType dt = in.getVariable(varName).getDataType();
- if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- continue; //skip rm instructions for non-matrix objects
-
- boolean foundRMInst = rContainsRMInstruction(pb, varName);
-
- if( !foundRMInst )
- {
- //create RMVAR instruction and put it into the programblock
- Instruction inst = createCleanupInstruction(varName);
-
- inst.setLocation(in.getVariable(varName));
-
- addCleanupInstruction(pb, inst);
-
- LOG.trace("Adding instruction (r1) "+inst.toString());
- }
- }
-
- //RULE 2: if in KILL and not in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- //(currently required for specific cases of nested loops)
- for( String varName : kill.getVariableNames() )
- if( (!in.containsVariable(varName)) && (!out.containsVariable(varName)) )
- {
- DataType dt = kill.getVariable(varName).getDataType();
- if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- continue; //skip rm instructions for non-matrix objects
-
- boolean foundRMInst = rContainsRMInstruction(pb, varName);
-
- if( !foundRMInst )
- {
- //create RMVAR instruction and put it into the programblock
- Instruction inst = createCleanupInstruction(varName);
-
- inst.setLocation(kill.getVariable(varName));
-
- //System.out.println("add rvar rule2 "+inst.toString());
- addCleanupInstruction(pb, inst);
-
- LOG.trace("Adding instruction (r2) "+inst.toString());
- }
- }
-
- return pb;
- }
-
- private Instruction createCleanupInstruction(String varName)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //(example "CP+Lops.OPERAND_DELIMITOR+rmvar+Lops.OPERAND_DELIMITOR+Var7")
- StringBuilder sb = new StringBuilder();
- sb.append("CP");
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append("rmvar");
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(varName);
- String str = sb.toString();
- Instruction inst = CPInstructionParser.parseSingleInstruction( str );
-
- return inst;
- }
-
- /**
- * Determines if the given program block includes a RMVAR or RMFILEVAR
- * instruction for the given varName.
- *
- * @param pb
- * @param varName
- * @return
- */
- private boolean rContainsRMInstruction(ProgramBlock pb, String varName)
- {
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock tmp = (WhileProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocks() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock tmp = (IfProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocksIfBody() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- for( ProgramBlock c : tmp.getChildBlocksElseBody() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock tmp = (ForProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocks() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if ( pb instanceof FunctionProgramBlock ) //includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
- {
- //do nothing
- }
- else
- {
- for( Instruction inst : pb.getInstructions() )
- {
- String instStr = inst.toString();
- if( instStr.contains("rmfilevar"+Lop.OPERAND_DELIMITOR+varName)
- || instStr.contains("rmvar"+Lop.OPERAND_DELIMITOR+varName) )
- {
- return true;
- }
- }
- }
-
-
- return false;
- }
-
- /**
- * Adds the generated cleanup RMVAR instruction to the given program block.
- * In case of generic (last-level) programblocks it is added to the end of
- * the list of instructions, while for complex program blocks it is added to
- * the end of the list of exit instructions.
- *
- * @param pb
- * @param inst
- * @throws DMLRuntimeException
- */
- private void addCleanupInstruction( ProgramBlock pb, Instruction inst )
- throws DMLRuntimeException
- {
- //System.out.println("Adding rm var instructions: "+inst.toString());
-
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock)pb;
- ArrayList<ProgramBlock> childs = wpb.getChildBlocks();
- if( !childs.get(childs.size()-1).getInstructions().isEmpty() ) //generic last level pb
- childs.get(childs.size()-1).addInstruction(inst);
- else{
- ProgramBlock pbNew = new ProgramBlock(pb.getProgram());
- pbNew.addInstruction(inst);
- childs.add(pbNew);
- }
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock wpb = (ForProgramBlock)pb;
- ArrayList<ProgramBlock> childs = wpb.getChildBlocks();
- if( !childs.get(childs.size()-1).getInstructions().isEmpty() ) //generic last level pb
- childs.get(childs.size()-1).addInstruction(inst);
- else{
- ProgramBlock pbNew = new ProgramBlock(pb.getProgram());
- pbNew.addInstruction(inst);
- childs.add(pbNew);
- }
- }
- else if (pb instanceof IfProgramBlock)
- ((IfProgramBlock)pb).addExitInstruction(inst);
- else if ( pb instanceof FunctionProgramBlock ) //includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
- ; //do nothing
- else
- {
- pb.addInstruction(inst); //add inst at end of pb
- }
- }
-
- /**
- *
- * @param fnamespace
- * @param fname
- * @return
- */
- public static String constructFunctionKey(String fnamespace, String fname)
- {
- return fnamespace + Program.KEY_DELIM + fname;
- }
-
- /**
- *
- * @param fkey
- * @return
- */
- public static String[] splitFunctionKey(String fkey)
- {
- return fkey.split(Program.KEY_DELIM);
- }
-}
-
[39/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/QuaternaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/QuaternaryOp.java b/src/main/java/com/ibm/bi/dml/hops/QuaternaryOp.java
deleted file mode 100644
index fd329f0..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/QuaternaryOp.java
+++ /dev/null
@@ -1,1684 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSigmoidR;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-
-/**
- * Note: this hop should be called AggQuaternaryOp in consistency with AggUnaryOp and AggBinaryOp;
- * however, since there does not exist a real QuaternaryOp yet - we can leave it as is for now.
- */
-public class QuaternaryOp extends Hop implements MultiThreadedHop
-{
-
- //config influencing mr operator selection (for testing purposes only)
- public static boolean FORCE_REPLICATION = false;
-
- private OpOp4 _op = null;
- private int _maxNumThreads = -1; //-1 for unlimited
-
- //wsloss-specific attributes
- private boolean _postWeights = false;
-
- //wsigmoid-specific attributes
- private boolean _logout = false;
- private boolean _minusin = false;
-
- //wdivmm-specific attributes
- private int _baseType = -1;
- private boolean _mult = false;
- private boolean _minus = false;
-
- //wumm-specific attributes
- private boolean _umult = false;
- private OpOp1 _uop = null;
- private OpOp2 _sop = null;
-
- private QuaternaryOp() {
- //default constructor for clone
- }
-
- /**
- * Constructor for wsloss.
- *
- * @param l
- * @param dt
- * @param vt
- * @param o
- * @param inX
- * @param inU
- * @param inV
- * @param inW
- * @param post
- */
- public QuaternaryOp(String l, DataType dt, ValueType vt, Hop.OpOp4 o,
- Hop inX, Hop inU, Hop inV, Hop inW, boolean post)
- {
- this(l, dt, vt, o, inX, inU, inV);
- getInput().add(3, inW);
- inW.getParent().add(this);
-
- _postWeights = post;
- }
-
- /**
- * Constructor for wsigmoid.
- *
- * @param l
- * @param dt
- * @param vt
- * @param o
- * @param inX
- * @param inU
- * @param inV
- * @param logout
- * @param minusin
- */
- public QuaternaryOp(String l, DataType dt, ValueType vt, Hop.OpOp4 o,
- Hop inX, Hop inU, Hop inV, boolean flag1, boolean flag2)
- {
- this(l, dt, vt, o, inX, inU, inV);
-
- _logout = flag1;
- _minusin = flag2;
- }
-
- public QuaternaryOp(String l, DataType dt, ValueType vt, Hop.OpOp4 o,
- Hop inW, Hop inU, Hop inV, int baseType, boolean flag1, boolean flag2)
- {
- this(l, dt, vt, o, inW, inU, inV);
-
- _baseType = baseType;
- _mult = flag1;
- _minus = flag2;
- }
-
- public QuaternaryOp(String l, DataType dt, ValueType vt, Hop.OpOp4 o,
- Hop inW, Hop inU, Hop inV, boolean umult, OpOp1 uop, OpOp2 sop)
- {
- this(l, dt, vt, o, inW, inU, inV);
-
- _umult = umult;
- _uop = uop;
- _sop = sop;
- }
-
- /**
- *
- * @param l
- * @param dt
- * @param vt
- * @param o
- * @param inX
- * @param inU
- * @param inV
- */
- public QuaternaryOp(String l, DataType dt, ValueType vt, Hop.OpOp4 o, Hop inX, Hop inU, Hop inV)
- {
- super(l, dt, vt);
- _op = o;
- getInput().add(0, inX);
- getInput().add(1, inU);
- getInput().add(2, inV);
- inX.getParent().add(this);
- inU.getParent().add(this);
- inV.getParent().add(this);
- }
-
- public OpOp4 getOp(){
- return _op;
- }
-
- @Override
- public void setMaxNumThreads( int k ) {
- _maxNumThreads = k;
- }
-
- @Override
- public int getMaxNumThreads() {
- return _maxNumThreads;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- try
- {
- ExecType et = optFindExecType();
-
- switch( _op ) {
- case WSLOSS: {
- WeightsType wtype = checkWeightsType();
-
- if( et == ExecType.CP )
- constructCPLopsWeightedSquaredLoss(wtype);
- else if( et == ExecType.MR )
- constructMRLopsWeightedSquaredLoss(wtype);
- else if( et == ExecType.SPARK )
- constructSparkLopsWeightedSquaredLoss(wtype);
- else
- throw new HopsException("Unsupported quaternaryop-wsloss exec type: "+et);
- break;
- }
-
- case WSIGMOID:{
- WSigmoidType wtype = checkWSigmoidType();
-
- if( et == ExecType.CP )
- constructCPLopsWeightedSigmoid(wtype);
- else if( et == ExecType.MR )
- constructMRLopsWeightedSigmoid(wtype);
- else if( et == ExecType.SPARK )
- constructSparkLopsWeightedSigmoid(wtype);
- else
- throw new HopsException("Unsupported quaternaryop-wsigmoid exec type: "+et);
- break;
- }
-
- case WDIVMM:{
- WDivMMType wtype = checkWDivMMType();
- if( et == ExecType.CP )
- constructCPLopsWeightedDivMM(wtype);
- else if( et == ExecType.MR )
- constructMRLopsWeightedDivMM(wtype);
- else if( et == ExecType.SPARK )
- constructSparkLopsWeightedDivMM(wtype);
- else
- throw new HopsException("Unsupported quaternaryop-wdivmm exec type: "+et);
- break;
- }
-
- case WCEMM:{
- WCeMMType wtype = WCeMMType.BASIC;
-
- if( et == ExecType.CP )
- constructCPLopsWeightedCeMM(wtype);
- else if( et == ExecType.MR )
- constructMRLopsWeightedCeMM(wtype);
- else if( et == ExecType.SPARK )
- constructSparkLopsWeightedCeMM(wtype);
- else
- throw new HopsException("Unsupported quaternaryop-wcemm exec type: "+et);
- break;
- }
-
- case WUMM:{
- WUMMType wtype = _umult ? WUMMType.MULT : WUMMType.DIV;
-
- if( et == ExecType.CP )
- constructCPLopsWeightedUMM(wtype);
- else if( et == ExecType.MR )
- constructMRLopsWeightedUMM(wtype);
- else if( et == ExecType.SPARK )
- constructSparkLopsWeightedUMM(wtype);
- else
- throw new HopsException("Unsupported quaternaryop-wumm exec type: "+et);
- break;
- }
-
- default:
- throw new HopsException(this.printErrorLocation() + "Unknown QuaternaryOp (" + _op + ") while constructing Lops");
- }
- }
- catch(LopsException e) {
- throw new HopsException(this.printErrorLocation() + "error constructing lops for QuaternaryOp." , e);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += "q(" + HopsOpOp4String.get(_op) + ")";
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + _op);
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsWeightedSquaredLoss(WeightsType wtype)
- throws HopsException, LopsException
- {
- WeightedSquaredLoss wsloss = new WeightedSquaredLoss(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getInput().get(3).constructLops(),
- getDataType(), getValueType(), wtype, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- wsloss.setNumThreads(k);
-
- setOutputDimensions( wsloss );
- setLineNumbers( wsloss );
- setLops( wsloss );
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsWeightedSquaredLoss(WeightsType wtype)
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsloss are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
- Hop W = getInput().get(3);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (!wtype.hasFourInputs() && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //partitioning of U
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
-
- //partitioning of V
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
-
- //map-side wsloss always with broadcast
- Lop wsloss = new WeightedSquaredLoss( X.constructLops(), lU, lV, W.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.MR);
- wsloss.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(wsloss);
-
- Group grp = new Group(wsloss, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum
- agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce())
- || (cacheU && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
-
- Group grpX = new Group(X.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpX.getOutputParameters().setDimensions(X.getDim1(), X.getDim2(), X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grpX);
-
- Lop grpW = W.constructLops();
- if( grpW.getDataType()==DataType.MATRIX ) {
- grpW = new Group(W.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpW.getOutputParameters().setDimensions(W.getDim1(), W.getDim2(), W.getRowsInBlock(), W.getColsInBlock(), -1);
- setLineNumbers(grpW);
- }
-
- Lop lU = null;
- if( cacheU ) {
- //partitioning of U for read through distributed cache
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
- }
- else {
- //replication of U for shuffle to target block
- Lop offset = createOffsetLop(V, false); //ncol of t(V) -> nrow of V determines num replicates
- lU = new RepMat(U.constructLops(), offset, true, V.getDataType(), V.getValueType());
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(),
- U.getRowsInBlock(), U.getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
-
- Group grpU = new Group(lU, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), U.getRowsInBlock(), U.getColsInBlock(), -1);
- setLineNumbers(grpU);
- lU = grpU;
- }
-
- Lop lV = null;
- if( cacheV ) {
- //partitioning of V for read through distributed cache
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
- }
- else {
- //replication of t(V) for shuffle to target block
- Transform ltV = new Transform( V.constructLops(), HopsTransf2Lops.get(ReOrgOp.TRANSPOSE), getDataType(), getValueType(), ExecType.MR);
- ltV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(ltV);
-
- Lop offset = createOffsetLop(U, false); //nrow of U determines num replicates
- lV = new RepMat(ltV, offset, false, V.getDataType(), V.getValueType());
- lV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(lV);
-
- Group grpV = new Group(lV, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(), V.getColsInBlock(), V.getRowsInBlock(), -1);
- setLineNumbers(grpV);
- lV = grpV;
- }
-
- //reduce-side wsloss w/ or without broadcast
- Lop wsloss = new WeightedSquaredLossR(
- grpX, lU, lV, grpW, DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.MR);
- wsloss.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(wsloss);
-
- Group grp = new Group(wsloss, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum
- agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsWeightedSquaredLoss(WeightsType wtype)
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsloss are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
- Hop W = getInput().get(3);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (!wtype.hasFourInputs() && m1Size+m2Size < memBudgetExec
- && 2*m1Size < memBudgetLocal && 2*m2Size < memBudgetLocal);
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //map-side wsloss always with broadcast
- Lop wsloss = new WeightedSquaredLoss( X.constructLops(), U.constructLops(), V.constructLops(), W.constructLops(),
- DataType.SCALAR, ValueType.DOUBLE, wtype, ExecType.SPARK);
- setOutputDimensions(wsloss);
- setLineNumbers(wsloss);
- setLops(wsloss);
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < memBudgetExec && 2*m1Size < memBudgetLocal);
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < memBudgetExec )
- || (cacheU && m1Size+m2Size < memBudgetExec)) && 2*m2Size < memBudgetLocal;
-
- //reduce-side wsloss w/ or without broadcast
- Lop wsloss = new WeightedSquaredLossR(
- X.constructLops(), U.constructLops(), V.constructLops(), W.constructLops(),
- DataType.SCALAR, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.SPARK);
- setOutputDimensions(wsloss);
- setLineNumbers(wsloss);
- setLops(wsloss);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsWeightedSigmoid(WSigmoidType wtype)
- throws HopsException, LopsException
- {
- WeightedSigmoid wsig = new WeightedSigmoid(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), wtype, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- wsig.setNumThreads(k);
-
- setOutputDimensions( wsig );
- setLineNumbers( wsig );
- setLops( wsig );
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsWeightedSigmoid( WSigmoidType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsigmoid are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //partitioning of U
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
-
- //partitioning of V
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
-
- //map-side wsloss always with broadcast
- Lop wsigmoid = new WeightedSigmoid( X.constructLops(), lU, lV,
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.MR);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops( wsigmoid );
-
- //in contrast to wsloss no aggregation required
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce())
- || (cacheU && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
-
- Group grpX = new Group(X.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpX.getOutputParameters().setDimensions(X.getDim1(), X.getDim2(), X.getRowsInBlock(), X.getColsInBlock(), X.getNnz());
- setLineNumbers(grpX);
-
- Lop lU = null;
- if( cacheU ) {
- //partitioning of U for read through distributed cache
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
- }
- else {
- //replication of U for shuffle to target block
- Lop offset = createOffsetLop(V, false); //ncol of t(V) -> nrow of V determines num replicates
- lU = new RepMat(U.constructLops(), offset, true, V.getDataType(), V.getValueType());
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(),
- U.getRowsInBlock(), U.getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
-
- Group grpU = new Group(lU, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), U.getRowsInBlock(), U.getColsInBlock(), -1);
- setLineNumbers(grpU);
- lU = grpU;
- }
-
- Lop lV = null;
- if( cacheV ) {
- //partitioning of V for read through distributed cache
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
- }
- else {
- //replication of t(V) for shuffle to target block
- Transform ltV = new Transform( V.constructLops(), HopsTransf2Lops.get(ReOrgOp.TRANSPOSE), getDataType(), getValueType(), ExecType.MR);
- ltV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(ltV);
-
- Lop offset = createOffsetLop(U, false); //nrow of U determines num replicates
- lV = new RepMat(ltV, offset, false, V.getDataType(), V.getValueType());
- lV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(lV);
-
- Group grpV = new Group(lV, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(), V.getColsInBlock(), V.getRowsInBlock(), -1);
- setLineNumbers(grpV);
- lV = grpV;
- }
-
- //reduce-side wsloss w/ or without broadcast
- Lop wsigmoid = new WeightedSigmoidR(
- grpX, lU, lV, DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.MR);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops(wsigmoid);
-
- //in contrast to wsloss no aggregation required
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsWeightedSigmoid( WSigmoidType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsigmoid are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < memBudgetExec
- && 2*m1Size<memBudgetLocal && 2*m2Size<memBudgetLocal);
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //map-side wsloss always with broadcast
- Lop wsigmoid = new WeightedSigmoid( X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.SPARK);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops( wsigmoid );
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < memBudgetExec && 2*m1Size < memBudgetLocal);
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < memBudgetExec )
- || (cacheU && m1Size+m2Size < memBudgetExec)) && 2*m2Size < memBudgetLocal;
-
- //reduce-side wsloss w/ or without broadcast
- Lop wsigmoid = new WeightedSigmoidR(
- X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.SPARK);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops(wsigmoid);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsWeightedDivMM(WDivMMType wtype)
- throws HopsException, LopsException
- {
- WeightedDivMM wdiv = new WeightedDivMM(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), wtype, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- wdiv.setNumThreads(k);
-
- setOutputDimensions( wdiv );
- setLineNumbers( wdiv );
- setLops( wdiv );
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsWeightedDivMM( WDivMMType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wdivmm are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Hop W = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //partitioning of U
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
-
- //partitioning of V
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
-
- //map-side wsloss always with broadcast
- Lop wdivmm = new WeightedDivMM( W.constructLops(), lU, lV,
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.MR);
- setOutputDimensions(wdivmm);
- setLineNumbers(wdivmm);
- setLops(wdivmm);
- }
- else //general case
- {
- //MR operator selection part 2 (both cannot happen for wdivmm, otherwise mapwdivmm)
- boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce())
- || (cacheU && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
-
- Group grpW = new Group(W.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpW.getOutputParameters().setDimensions(W.getDim1(), W.getDim2(), W.getRowsInBlock(), W.getColsInBlock(), W.getNnz());
- setLineNumbers(grpW);
-
- Lop lU = null;
- if( cacheU ) {
- //partitioning of U for read through distributed cache
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
- }
- else {
- //replication of U for shuffle to target block
- Lop offset = createOffsetLop(V, false); //ncol of t(V) -> nrow of V determines num replicates
- lU = new RepMat(U.constructLops(), offset, true, V.getDataType(), V.getValueType());
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(),
- U.getRowsInBlock(), U.getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
-
- Group grpU = new Group(lU, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), U.getRowsInBlock(), U.getColsInBlock(), -1);
- setLineNumbers(grpU);
- lU = grpU;
- }
-
- Lop lV = null;
- if( cacheV ) {
- //partitioning of V for read through distributed cache
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
- }
- else {
- //replication of t(V) for shuffle to target block
- Transform ltV = new Transform( V.constructLops(), HopsTransf2Lops.get(ReOrgOp.TRANSPOSE), getDataType(), getValueType(), ExecType.MR);
- ltV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(ltV);
-
- Lop offset = createOffsetLop(U, false); //nrow of U determines num replicates
- lV = new RepMat(ltV, offset, false, V.getDataType(), V.getValueType());
- lV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(lV);
-
- Group grpV = new Group(lV, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(), V.getColsInBlock(), V.getRowsInBlock(), -1);
- setLineNumbers(grpV);
- lV = grpV;
- }
-
- //reduce-side wsloss w/ or without broadcast
- Lop wdivmm = new WeightedDivMMR( grpW, lU, lV,
- DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.MR);
- setOutputDimensions(wdivmm);
- setLineNumbers(wdivmm);
- setLops(wdivmm);
- }
-
- //in contrast to to wsloss/wsigmoid, wdivmm requires partial aggregation (for the final mm)
- Group grp = new Group(getLops(), Group.OperationTypes.Sort, getDataType(), getValueType());
- setOutputDimensions(grp);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), getDataType(), getValueType(), ExecType.MR);
- // aggregation uses kahanSum but the inputs do not have correction values
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
-
- setLops(agg1);
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsWeightedDivMM( WDivMMType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wdivmm are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted DivMM only if this constraint holds.
-
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- Hop W = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < memBudgetExec
- && 2*m1Size<memBudgetLocal && 2*m2Size<memBudgetLocal);
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //map-side wsloss always with broadcast
- Lop wdivmm = new WeightedDivMM( W.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.SPARK);
- setOutputDimensions(wdivmm);
- setLineNumbers(wdivmm);
- setLops( wdivmm );
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < memBudgetExec && 2*m1Size < memBudgetLocal);
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < memBudgetExec )
- || (cacheU && m1Size+m2Size < memBudgetExec)) && 2*m2Size < memBudgetLocal;
-
- //reduce-side wsloss w/ or without broadcast
- Lop wdivmm = new WeightedDivMMR(
- W.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.SPARK);
- setOutputDimensions(wdivmm);
- setLineNumbers(wdivmm);
- setLops(wdivmm);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsWeightedCeMM(WCeMMType wtype)
- throws HopsException, LopsException
- {
- WeightedCrossEntropy wcemm = new WeightedCrossEntropy(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), wtype, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- wcemm.setNumThreads(k);
-
- setOutputDimensions( wcemm );
- setLineNumbers( wcemm );
- setLops( wcemm );
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsWeightedCeMM(WCeMMType wtype)
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsloss are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //partitioning of U
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
-
- //partitioning of V
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
-
- //map-side wsloss always with broadcast
- Lop wcemm = new WeightedCrossEntropy( X.constructLops(), lU, lV,
- DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.MR);
- wcemm.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(wcemm);
-
- Group grp = new Group(wcemm, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum
- agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce())
- || (cacheU && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
-
- Group grpX = new Group(X.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpX.getOutputParameters().setDimensions(X.getDim1(), X.getDim2(), X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grpX);
-
- Lop lU = null;
- if( cacheU ) {
- //partitioning of U for read through distributed cache
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
- }
- else {
- //replication of U for shuffle to target block
- Lop offset = createOffsetLop(V, false); //ncol of t(V) -> nrow of V determines num replicates
- lU = new RepMat(U.constructLops(), offset, true, V.getDataType(), V.getValueType());
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(),
- U.getRowsInBlock(), U.getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
-
- Group grpU = new Group(lU, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), U.getRowsInBlock(), U.getColsInBlock(), -1);
- setLineNumbers(grpU);
- lU = grpU;
- }
-
- Lop lV = null;
- if( cacheV ) {
- //partitioning of V for read through distributed cache
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
- }
- else {
- //replication of t(V) for shuffle to target block
- Transform ltV = new Transform( V.constructLops(), HopsTransf2Lops.get(ReOrgOp.TRANSPOSE), getDataType(), getValueType(), ExecType.MR);
- ltV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(ltV);
-
- Lop offset = createOffsetLop(U, false); //nrow of U determines num replicates
- lV = new RepMat(ltV, offset, false, V.getDataType(), V.getValueType());
- lV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(lV);
-
- Group grpV = new Group(lV, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(), V.getColsInBlock(), V.getRowsInBlock(), -1);
- setLineNumbers(grpV);
- lV = grpV;
- }
-
- //reduce-side wsloss w/ or without broadcast
- Lop wcemm = new WeightedCrossEntropyR(
- grpX, lU, lV, DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.MR);
- wcemm.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(wcemm);
-
- Group grp = new Group(wcemm, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum
- agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsWeightedCeMM(WCeMMType wtype)
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsloss are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < memBudgetExec
- && 2*m1Size < memBudgetLocal && 2*m2Size < memBudgetLocal);
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //map-side wsloss always with broadcast
- Lop wsloss = new WeightedCrossEntropy( X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.SCALAR, ValueType.DOUBLE, wtype, ExecType.SPARK);
- setOutputDimensions(wsloss);
- setLineNumbers(wsloss);
- setLops(wsloss);
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < memBudgetExec && 2*m1Size < memBudgetLocal);
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < memBudgetExec )
- || (cacheU && m1Size+m2Size < memBudgetExec)) && 2*m2Size < memBudgetLocal;
-
- //reduce-side wsloss w/ or without broadcast
- Lop wcemm = new WeightedCrossEntropyR(
- X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.SCALAR, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.SPARK);
- setOutputDimensions(wcemm);
- setLineNumbers(wcemm);
- setLops(wcemm);
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsWeightedUMM(WUMMType wtype)
- throws HopsException, LopsException
- {
- Unary.OperationTypes uop = _uop!=null ?
- HopsOpOp1LopsU.get(_uop) : _sop==OpOp2.POW ?
- Unary.OperationTypes.POW2 : Unary.OperationTypes.MULTIPLY2;
-
- WeightedUnaryMM wsig = new WeightedUnaryMM(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), wtype, uop, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- wsig.setNumThreads(k);
-
- setOutputDimensions( wsig );
- setLineNumbers( wsig );
- setLops( wsig );
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsWeightedUMM( WUMMType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsigmoid are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Unary.OperationTypes uop = _uop!=null ?
- HopsOpOp1LopsU.get(_uop) : _sop==OpOp2.POW ?
- Unary.OperationTypes.POW2 : Unary.OperationTypes.MULTIPLY2;
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //partitioning of U
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
-
- //partitioning of V
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
-
- //map-side wsloss always with broadcast
- Lop wumm = new WeightedUnaryMM( X.constructLops(), lU, lV,
- DataType.MATRIX, ValueType.DOUBLE, wtype, uop, ExecType.MR);
- setOutputDimensions(wumm);
- setLineNumbers(wumm);
- setLops( wumm );
-
- //in contrast to wsloss no aggregation required
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce())
- || (cacheU && m1Size+m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
-
- Group grpX = new Group(X.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpX.getOutputParameters().setDimensions(X.getDim1(), X.getDim2(), X.getRowsInBlock(), X.getColsInBlock(), X.getNnz());
- setLineNumbers(grpX);
-
- Lop lU = null;
- if( cacheU ) {
- //partitioning of U for read through distributed cache
- boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lU = U.constructLops();
- if( needPartU ){ //requires partitioning
- lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
- }
- }
- else {
- //replication of U for shuffle to target block
- Lop offset = createOffsetLop(V, false); //ncol of t(V) -> nrow of V determines num replicates
- lU = new RepMat(U.constructLops(), offset, true, V.getDataType(), V.getValueType());
- lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(),
- U.getRowsInBlock(), U.getColsInBlock(), U.getNnz());
- setLineNumbers(lU);
-
- Group grpU = new Group(lU, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), U.getRowsInBlock(), U.getColsInBlock(), -1);
- setLineNumbers(grpU);
- lU = grpU;
- }
-
- Lop lV = null;
- if( cacheV ) {
- //partitioning of V for read through distributed cache
- boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- lV = V.constructLops();
- if( needPartV ){ //requires partitioning
- lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
- setLineNumbers(lV);
- }
- }
- else {
- //replication of t(V) for shuffle to target block
- Transform ltV = new Transform( V.constructLops(), HopsTransf2Lops.get(ReOrgOp.TRANSPOSE), getDataType(), getValueType(), ExecType.MR);
- ltV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(ltV);
-
- Lop offset = createOffsetLop(U, false); //nrow of U determines num replicates
- lV = new RepMat(ltV, offset, false, V.getDataType(), V.getValueType());
- lV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(),
- V.getColsInBlock(), V.getRowsInBlock(), V.getNnz());
- setLineNumbers(lV);
-
- Group grpV = new Group(lV, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
- grpV.getOutputParameters().setDimensions(V.getDim2(), V.getDim1(), V.getColsInBlock(), V.getRowsInBlock(), -1);
- setLineNumbers(grpV);
- lV = grpV;
- }
-
- //reduce-side wsloss w/ or without broadcast
- Lop wumm = new WeightedUnaryMMR(
- grpX, lU, lV, DataType.MATRIX, ValueType.DOUBLE, wtype, uop, cacheU, cacheV, ExecType.MR);
- setOutputDimensions(wumm);
- setLineNumbers(wumm);
- setLops(wumm);
-
- //in contrast to wsloss no aggregation required
- }
- }
-
- /**
- *
- * @param wtype
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsWeightedUMM( WUMMType wtype )
- throws HopsException, LopsException
- {
- //NOTE: the common case for wsigmoid are factors U/V with a rank of 10s to 100s; the current runtime only
- //supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
- //by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
-
- Unary.OperationTypes uop = _uop!=null ?
- HopsOpOp1LopsU.get(_uop) : _sop==OpOp2.POW ?
- Unary.OperationTypes.POW2 : Unary.OperationTypes.MULTIPLY2;
-
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- Hop X = getInput().get(0);
- Hop U = getInput().get(1);
- Hop V = getInput().get(2);
-
- //MR operator selection, part1
- double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2()); //size U
- double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2()); //size V
- boolean isMapWsloss = (m1Size+m2Size < memBudgetExec
- && 2*m1Size<memBudgetLocal && 2*m2Size<memBudgetLocal);
-
- if( !FORCE_REPLICATION && isMapWsloss ) //broadcast
- {
- //map-side wsloss always with broadcast
- Lop wsigmoid = new WeightedUnaryMM( X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, uop, ExecType.SPARK);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops( wsigmoid );
- }
- else //general case
- {
- //MR operator selection part 2
- boolean cacheU = !FORCE_REPLICATION && (m1Size < memBudgetExec && 2*m1Size < memBudgetLocal);
- boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < memBudgetExec )
- || (cacheU && m1Size+m2Size < memBudgetExec)) && 2*m2Size < memBudgetLocal;
-
- //reduce-side wsloss w/ or without broadcast
- Lop wsigmoid = new WeightedUnaryMMR(
- X.constructLops(), U.constructLops(), V.constructLops(),
- DataType.MATRIX, ValueType.DOUBLE, wtype, uop, cacheU, cacheV, ExecType.SPARK);
- setOutputDimensions(wsigmoid);
- setLineNumbers(wsigmoid);
- setLops(wsigmoid);
- }
- }
-
- /**
- *
- * @return
- */
- private WeightsType checkWeightsType()
- {
- WeightsType ret = WeightsType.NONE;
- if( !(getInput().get(3) instanceof LiteralOp) ){
- if( _postWeights )
- ret = WeightsType.POST;
- else
- ret = WeightsType.PRE;
- }
- else if( _postWeights ){
- ret = WeightsType.POST_NZ;
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- private WSigmoidType checkWSigmoidType()
- {
-
- if( _logout && _minusin )
- return WSigmoidType.LOG_MINUS;
- else if( _logout )
- return WSigmoidType.LOG;
- else if( _minusin )
- return WSigmoidType.MINUS;
- else
- return WSigmoidType.BASIC;
- }
-
- /**
- *
- * @return
- */
- private WDivMMType checkWDivMMType()
- {
- switch( _baseType )
- {
- case 0: //BASIC
- return WDivMMType.MULT_BASIC;
- case 1: //LEFT
- if( _minus )
- return WDivMMType.MULT_MINUS_LEFT;
- else
- return _mult ? WDivMMType.MULT_LEFT : WDivMMType.DIV_LEFT;
- case 2: //RIGHT
- if( _minus )
- return WDivMMType.MULT_MINUS_RIGHT;
- else
- return _mult ? WDivMMType.MULT_RIGHT : WDivMMType.DIV_RIGHT;
- }
-
- return null;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- switch( _op ) {
- case WSLOSS: //always scalar output
- case WCEMM:
- return OptimizerUtils.DOUBLE_SIZE;
-
- case WSIGMOID:
- case WDIVMM:
- case WUMM:
- double sp = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sp);
-
- default:
- return 0;
- }
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- //no intermediates
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- switch( _op ) {
- case WSLOSS: //always scalar output
- ret = null;
- break;
-
- case WSIGMOID:
- case WUMM: {
- MatrixCharacteristics mcW = memo.getAllInputStats(getInput().get(0));
- ret = new long[]{mcW.getRows(), mcW.getCols(), mcW.getNonZeros()};
- break;
- }
-
- case WDIVMM: {
- if( _baseType == 0 ){ //basic
- MatrixCharacteristics mcW = memo.getAllInputStats(getInput().get(0));
- ret = new long[]{mcW.getRows(), mcW.getCols(), mcW.getNonZeros()};
- }
- if( _baseType == 1 ) { //left (w/ transpose)
- MatrixCharacteristics mcV = memo.getAllInputStats(getInput().get(2));
- ret = new long[]{mcV.getRows(), mcV.getCols(), -1};
- }
- else { //right
- MatrixCharacteristics mcU = memo.getAllInputStats(getInput().get(1));
- ret = new long[]{mcU.getRows(), mcU.getCols(), -1};
- }
- break;
- }
-
- default:
- throw new RuntimeException("Memory for operation (" + _op + ") can not be estimated.");
- }
-
- return ret;
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if ( (getInput().get(0).areDimsBelowThreshold()
- && getInput().get(1).areDimsBelowThreshold()
- && getInput().get(2).areDimsBelowThreshold()
- && getInput().get(3).areDimsBelowThreshold()) )
- _etype = ExecType.CP;
- else
- _etype = REMOTE;
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- switch( _op ) {
- case WSLOSS:
- //do nothing: always scalar
- break;
-
- case WSIGMOID:
- case WUMM: {
- Hop inW = getInput().get(0);
- setDim1( inW.getDim1() );
- setDim2( inW.getDim2() );
- setNnz( inW.getNnz() );
- break;
- }
-
- case WDIVMM: {
- if( _baseType == 0 ) { //basic
- Hop inW = getInput().get(0);
- setDim1( inW.getDim1() );
- setDim2( inW.getDim2() );
- setNnz( inW.getNnz() );
- }
- else if( _baseType == 1 ){ //left (w/ transpose)
- Hop inV = getInput().get(2);
- setDim1( inV.getDim1() );
- setDim2( inV.getDim2() );
- }
- else { //right
- Hop inU = getInput().get(1);
- setDim1( inU.getDim1() );
- setDim2( inU.getDim2() );
- }
- break;
- }
-
- default:
- break;
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- QuaternaryOp ret = new QuaternaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._postWeights = _postWeights;
- ret._logout = _logout;
- ret._minusin = _minusin;
- ret._baseType = _baseType;
- ret._mult = _mult;
- ret._minus = _minus;
- ret._umult = _umult;
- ret._uop = _uop;
- ret._sop = _sop;
- ret._maxNumThreads = _maxNumThreads;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof QuaternaryOp) )
- return false;
-
- QuaternaryOp that2 = (QuaternaryOp)that;
-
- //compare basic inputs and weights (always existing)
- boolean ret = (_op == that2._op
- && getInput().size() == getInput().size()
- && getInput().get(0) == that2.getInput().get(0)
- && getInput().get(1) == that2.getInput().get(1)
- && getInput().get(2) == that2.getInput().get(2) );
-
- //check for 4th argument if same size (see above)
- if( ret && getInput().size()==4 )
- ret &= (getInput().get(3) == that2.getInput().get(3));
-
- //compare specific parameters
- ret &= _postWeights == that2._postWeights;
- ret &= _logout == that2._logout;
- ret &= _minusin == that2._minusin;
- ret &= _baseType == that2._baseType;
- ret &= _mult == that2._mult;
- ret &= _minus == that2._minus;
- ret &= _umult == that2._umult;
- ret &= _uop == that2._uop;
- ret &= _sop == that2._sop;
- ret &= _maxNumThreads == that2._maxNumThreads;
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/ReorgOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/ReorgOp.java b/src/main/java/com/ibm/bi/dml/hops/ReorgOp.java
deleted file mode 100644
index 703fb4a..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/ReorgOp.java
+++ /dev/null
@@ -1,608 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-/**
- * Reorg (cell) operation: aij
- * Properties:
- * Symbol: ', rdiag, rshape, rsort
- * 1 Operand
- *
- * Semantic: change indices (in mapper or reducer)
- *
- *
- * NOTE MB: reshape integrated here because (1) ParameterizedBuiltinOp requires name-value pairs for params
- * and (2) most importantly semantic of reshape is exactly a reorg op.
- */
-
-public class ReorgOp extends Hop
-{
-
- public static boolean FORCE_DIST_SORT_INDEXES = false;
-
- public boolean bSortSPRewriteApplicable = false;
-
- private ReOrgOp op;
-
- private ReorgOp() {
- //default constructor for clone
- }
-
- public ReorgOp(String l, DataType dt, ValueType vt, ReOrgOp o, Hop inp)
- {
- super(l, dt, vt);
- op = o;
- getInput().add(0, inp);
- inp.getParent().add(this);
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- public ReorgOp(String l, DataType dt, ValueType vt, ReOrgOp o, ArrayList<Hop> inp)
- {
- super(l, dt, vt);
- op = o;
-
- for( int i=0; i<inp.size(); i++ ) {
- Hop in = inp.get(i);
- getInput().add(i, in);
- in.getParent().add(this);
- }
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- public ReOrgOp getOp()
- {
- return op;
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += "r(" + HopsTransf2String.get(op) + ")";
- return s;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
-
- switch( op )
- {
- case TRANSPOSE:
- case DIAG:
- {
- Transform transform1 = new Transform( getInput().get(0).constructLops(),
- HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
- setOutputDimensions(transform1);
- setLineNumbers(transform1);
- setLops(transform1);
-
- break;
- }
- case RESHAPE:
- {
- if( et==ExecType.MR )
- {
- Transform transform1 = new Transform( getInput().get(0).constructLops(),
- HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
- setOutputDimensions(transform1);
- setLineNumbers(transform1);
- for( int i=1; i<=3; i++ ) //rows, cols, byrow
- {
- Lop ltmp = getInput().get(i).constructLops();
- transform1.addInput(ltmp);
- ltmp.addOutput(transform1);
- }
- transform1.setLevel(); //force order of added lops
-
- Group group1 = new Group(
- transform1, Group.OperationTypes.Sort, DataType.MATRIX,
- getValueType());
- setOutputDimensions(group1);
- setLineNumbers(group1);
-
- Aggregate agg1 = new Aggregate(
- group1, Aggregate.OperationTypes.Sum, DataType.MATRIX,
- getValueType(), et);
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
-
- setLops(agg1);
- }
- else //CP/SPARK
- {
- Transform transform1 = new Transform( getInput().get(0).constructLops(),
- HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
- setOutputDimensions(transform1);
- setLineNumbers(transform1);
-
- for( int i=1; i<=3; i++ ) //rows, cols, byrow
- {
- Lop ltmp = getInput().get(i).constructLops();
- transform1.addInput(ltmp);
- ltmp.addOutput(transform1);
- }
- transform1.setLevel(); //force order of added lops
-
- setLops(transform1);
- }
- break;
- }
- case SORT:
- {
- Hop input = getInput().get(0);
- Hop by = getInput().get(1);
- Hop desc = getInput().get(2);
- Hop ixret = getInput().get(3);
-
- if( et==ExecType.MR )
- {
-
- if( !(desc instanceof LiteralOp && ixret instanceof LiteralOp) ) {
- LOG.warn("Unsupported non-constant ordering parameters, using defaults and mark for recompilation.");
- setRequiresRecompile();
- desc = new LiteralOp(false);
- ixret = new LiteralOp(false);
- }
-
- //Step 1: extraction (if unknown ncol or multiple columns)
- Hop vinput = input;
- if( input.getDim2() != 1 ) {
- vinput = new IndexingOp("tmp1", getDataType(), getValueType(), input, new LiteralOp(1L),
- HopRewriteUtils.createValueHop(input, true), by, by, false, true);
- vinput.refreshSizeInformation();
- HopRewriteUtils.setOutputBlocksizes(vinput, getRowsInBlock(), getColsInBlock());
- HopRewriteUtils.copyLineNumbers(this, vinput);
- }
-
- //Step 2: Index vector sort
- Hop voutput = null;
- if( 2*OptimizerUtils.estimateSize(vinput.getDim1(), vinput.getDim2())
- > OptimizerUtils.getLocalMemBudget()
- || FORCE_DIST_SORT_INDEXES )
- {
- //large vector, fallback to MR sort
- //sort indexes according to given values
- SortKeys sort = new SortKeys(
- vinput.constructLops(), HopRewriteUtils.getBooleanValueSafe((LiteralOp)desc),
- SortKeys.OperationTypes.Indexes,
- vinput.getDataType(), vinput.getValueType(), ExecType.MR);
-
- sort.getOutputParameters().setDimensions(vinput.getDim1(), 1,
- vinput.getRowsInBlock(), vinput.getColsInBlock(), vinput.getNnz());
-
- setLineNumbers(sort);
-
- //note: this sortindexes includes also the shift by offsets and
- //final aggregate because sideways passing of offsets would
- //not nicely fit the current instruction model
-
- setLops(sort);
- voutput = this;
- }
- else
- {
- //small vector, use in-memory sort
- ArrayList<Hop> sinputs = new ArrayList<Hop>();
- sinputs.add(vinput);
- sinputs.add(new LiteralOp(1)); //by (always vector)
- sinputs.add(desc);
- sinputs.add(new LiteralOp(true)); //indexreturn (always indexes)
- voutput = new ReorgOp("tmp3", getDataType(), getValueType(), ReOrgOp.SORT, sinputs);
- HopRewriteUtils.copyLineNumbers(this, voutput);
- //explicitly construct CP lop; otherwise there is danger of infinite recursion if forced runtime platform.
- voutput.setLops( constructCPOrSparkSortLop(vinput, sinputs.get(1), sinputs.get(2), sinputs.get(3), ExecType.CP, false) );
- voutput.getLops().getOutputParameters().setDimensions(vinput.getDim1(), vinput.getDim2(), vinput.getRowsInBlock(), vinput.getColsInBlock(), vinput.getNnz());
- setLops( voutput.constructLops() );
- }
-
- //Step 3: Data permutation (only required for sorting data)
- // -- done via X' = table(seq(), IX') %*% X;
- if( !HopRewriteUtils.getBooleanValueSafe((LiteralOp)ixret) )
- {
- //generate seq
- DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(voutput);
- seq.setName("tmp4");
- seq.refreshSizeInformation();
- seq.computeMemEstimate(new MemoTable()); //select exec type
- HopRewriteUtils.copyLineNumbers(this, seq);
-
- //generate table
- TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, seq, voutput, new LiteralOp(1L) );
- HopRewriteUtils.setOutputBlocksizes(table, getRowsInBlock(), getColsInBlock());
- table.refreshSizeInformation();
- table.setForcedExecType(ExecType.MR); //force MR
- HopRewriteUtils.copyLineNumbers(this, table);
- table.setDisjointInputs(true);
- table.setOutputEmptyBlocks(false);
-
- //generate matrix mult
- AggBinaryOp mmult = HopRewriteUtils.createMatrixMultiply(table, input);
- mmult.setForcedExecType(ExecType.MR); //force MR
-
- setLops( mmult.constructLops() );
-
- //cleanups
- HopRewriteUtils.removeChildReference(table, input);
- }
- }
- else //CP or Spark
- {
- if( et==ExecType.SPARK && !FORCE_DIST_SORT_INDEXES)
- bSortSPRewriteApplicable = isSortSPRewriteApplicable();
-
- Lop transform1 = constructCPOrSparkSortLop(input, by, desc, ixret, et, bSortSPRewriteApplicable);
- setOutputDimensions(transform1);
- setLineNumbers(transform1);
-
- setLops(transform1);
- }
- break;
- }
-
- default:
- throw new HopsException("Unsupported lops construction for operation type '"+op+"'.");
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- private static Lop constructCPOrSparkSortLop( Hop input, Hop by, Hop desc, Hop ixret, ExecType et, boolean bSortIndInMem )
- throws HopsException, LopsException
- {
- Transform transform1 = new Transform( input.constructLops(), HopsTransf2Lops.get(ReOrgOp.SORT),
- input.getDataType(), input.getValueType(), et, bSortIndInMem);
-
- for( Hop c : new Hop[]{by,desc,ixret} ) {
- Lop ltmp = c.constructLops();
- transform1.addInput(ltmp);
- ltmp.addOutput(transform1);
- }
-
- transform1.setLevel(); //force order of added lops
-
- return transform1;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- //no dedicated mem estimation per op type, because always propagated via refreshSizeInformation
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- if( op == ReOrgOp.SORT )
- {
- Hop ixreturn = getInput().get(3);
- if( !(ixreturn instanceof LiteralOp && !HopRewriteUtils.getBooleanValueSafe((LiteralOp)ixreturn)
- && (dim2==1 || nnz==0) ) ) //NOT early abort case
- {
- //Version 2: memory requirements for temporary index int[] array,
- //(temporary double[] array already covered by output)
- return dim1 * 4;
-
- //Version 1: memory requirements for temporary index Integer[] array
- //8-16 (12) bytes for object, 4byte int payload, 4-8 (8) byte pointers.
- //return dim1 * 24;
- }
- }
-
- //default: no intermediate memory requirements
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- Hop input = getInput().get(0);
- MatrixCharacteristics mc = memo.getAllInputStats(input);
-
- switch(op)
- {
- case TRANSPOSE:
- {
- // input is a [k1,k2] matrix and output is a [k2,k1] matrix
- // #nnz in output is exactly the same as in input
- if( mc.dimsKnown() )
- ret = new long[]{ mc.getCols(), mc.getRows(), mc.getNonZeros() };
- break;
- }
- case DIAG:
- {
- // NOTE: diag is overloaded according to the number of columns of the input
-
- long k = mc.getRows();
-
- // CASE a) DIAG V2M
- // input is a [1,k] or [k,1] matrix, and output is [k,k] matrix
- // #nnz in output is in the worst case k => sparsity = 1/k
- if( k == 1 )
- ret = new long[]{k, k, ((mc.getNonZeros()>=0) ? mc.getNonZeros() : k)};
-
- // CASE b) DIAG M2V
- // input is [k,k] matrix and output is [k,1] matrix
- // #nnz in the output is likely to be k (a dense matrix)
- if( k > 1 )
- ret = new long[]{k, 1, ((mc.getNonZeros()>=0) ? Math.min(k,mc.getNonZeros()) : k) };
-
- break;
- }
- case RESHAPE:
- {
- // input is a [k1,k2] matrix and output is a [k3,k4] matrix with k1*k2=k3*k4
- // #nnz in output is exactly the same as in input
- if( mc.dimsKnown() ) {
- if( _dim1 > 0 )
- ret = new long[]{ _dim1, mc.getRows()*mc.getCols()/_dim1, mc.getNonZeros()};
- else if( _dim2 > 0 )
- ret = new long[]{ mc.getRows()*mc.getCols()/_dim2, _dim2, mc.getNonZeros()};
- }
- break;
- }
- case SORT:
- {
- // input is a [k1,k2] matrix and output is a [k1,k3] matrix, where k3=k2 if no index return;
- // otherwise k3=1 (for the index vector)
- Hop input4 = getInput().get(3); //indexreturn
- boolean unknownIxRet = !(input4 instanceof LiteralOp);
-
- if( !unknownIxRet ) {
- boolean ixret = HopRewriteUtils.getBooleanValueSafe((LiteralOp)input4);
- long dim2 = ixret ? 1 : mc.getCols();
- long nnz = ixret ? mc.getRows() : mc.getNonZeros();
- ret = new long[]{ mc.getRows(), dim2, nnz};
- }
- else {
- ret = new long[]{ mc.getRows(), -1, -1};
- }
- }
- }
-
- return ret;
- }
-
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- // Choose CP, if the input dimensions are below threshold or if the input is a vector
- else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = getInput().get(0);
-
- switch(op)
- {
- case TRANSPOSE:
- {
- // input is a [k1,k2] matrix and output is a [k2,k1] matrix
- // #nnz in output is exactly the same as in input
- setDim1(input1.getDim2());
- setDim2(input1.getDim1());
- setNnz(input1.getNnz());
- break;
- }
- case DIAG:
- {
- // NOTE: diag is overloaded according to the number of columns of the input
-
- long k = input1.getDim1();
- setDim1(k);
-
- // CASE a) DIAG_V2M
- // input is a [1,k] or [k,1] matrix, and output is [k,k] matrix
- // #nnz in output is in the worst case k => sparsity = 1/k
- if( input1.getDim2()==1 ) {
- setDim2(k);
- setNnz( (input1.getNnz()>=0) ? input1.getNnz() : k );
- }
-
- // CASE b) DIAG_M2V
- // input is [k,k] matrix and output is [k,1] matrix
- // #nnz in the output is likely to be k (a dense matrix)
- if( input1.getDim2()>1 ){
- setDim2(1);
- setNnz( (input1.getNnz()>=0) ? Math.min(k,input1.getNnz()) : k );
- }
-
- break;
- }
- case RESHAPE:
- {
- // input is a [k1,k2] matrix and output is a [k3,k4] matrix with k1*k2=k3*k4
- // #nnz in output is exactly the same as in input
- Hop input2 = getInput().get(1); //rows
- Hop input3 = getInput().get(2); //cols
- refreshRowsParameterInformation(input2); //refresh rows
- refreshColsParameterInformation(input3); //refresh cols
- setNnz(input1.getNnz());
- if( !dimsKnown() &&input1.dimsKnown() ) { //reshape allows to infer dims, if input and 1 dim known
- if(_dim1 > 0)
- _dim2 = (input1._dim1*input1._dim2)/_dim1;
- else if(_dim2 > 0)
- _dim1 = (input1._dim1*input1._dim2)/_dim2;
- }
- break;
- }
- case SORT:
- {
- // input is a [k1,k2] matrix and output is a [k1,k3] matrix, where k3=k2 if no index return;
- // otherwise k3=1 (for the index vector)
- Hop input4 = getInput().get(3); //indexreturn
- boolean unknownIxRet = !(input4 instanceof LiteralOp);
-
- _dim1 = input1.getDim1();
- if( !unknownIxRet ) {
- boolean ixret = HopRewriteUtils.getBooleanValueSafe((LiteralOp)input4);
- _dim2 = ixret ? 1 : input1.getDim2();
- _nnz = ixret ? input1.getDim1() : input1.getNnz();
- }
- else {
- _dim2 = -1;
- _nnz = -1;
- }
- break;
- }
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- ReorgOp ret = new ReorgOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret.op = op;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof ReorgOp) )
- return false;
-
- ReorgOp that2 = (ReorgOp)that;
- boolean ret = (op == that2.op)
- && (getInput().size()==that.getInput().size());
-
- //compare all childs (see reshape, sort)
- if( ret ) //sizes matched
- for( int i=0; i<_input.size(); i++ )
- ret &= getInput().get(i) == that2.getInput().get(i);
-
- return ret;
- }
-
-
- @Override
- public void printMe() throws HopsException
- {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + op);
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- /**
- * This will check if there is sufficient memory locally (twice the size of second matrix, for original and sort data), and remotely (size of second matrix (sorted data)).
- * @return
- */
- private boolean isSortSPRewriteApplicable()
- {
- boolean ret = false;
- Hop input = getInput().get(0);
-
- //note: both cases (partitioned matrix, and sorted double array), require to
- //fit the broadcast twice into the local memory budget. Also, the memory
- //constraint only needs to take the rhs into account because the output is
- //guaranteed to be an aggregate of <=16KB
-
- double size = input.dimsKnown() ?
- OptimizerUtils.estimateSize(input.getDim1(), 1) : //dims known and estimate fits
- input.getOutputMemEstimate(); //dims unknown but worst-case estimate fits
-
- if( OptimizerUtils.checkSparkBroadcastMemoryBudget(size) ) {
- ret = true;
- }
-
- return ret;
- }
-}
\ No newline at end of file
[60/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
index b59112a..8e97a28 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.Serializable;
import java.util.NoSuchElementException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/IndexRange.java b/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
index b38702a..afa70e8 100644
--- a/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
+++ b/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
index 8041f06..7827e50 100644
--- a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.File;
import java.io.FileInputStream;
@@ -23,17 +23,17 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.Pair;
public class LocalFileUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/LongLongDoubleHashMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/LongLongDoubleHashMap.java b/src/main/java/org/apache/sysml/runtime/util/LongLongDoubleHashMap.java
index a40bf14..d7f5831 100644
--- a/src/main/java/org/apache/sysml/runtime/util/LongLongDoubleHashMap.java
+++ b/src/main/java/org/apache/sysml/runtime/util/LongLongDoubleHashMap.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
index 63c58a7..3b37c39 100644
--- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
+++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -37,20 +37,20 @@ import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.sort.ReadWithZeros;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.sort.ReadWithZeros;
public class MapReduceTool
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/NormalPRNGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/NormalPRNGenerator.java b/src/main/java/org/apache/sysml/runtime/util/NormalPRNGenerator.java
index a8050a5..82e85c1 100644
--- a/src/main/java/org/apache/sysml/runtime/util/NormalPRNGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/util/NormalPRNGenerator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/PRNGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/PRNGenerator.java b/src/main/java/org/apache/sysml/runtime/util/PRNGenerator.java
index 6dc2631..0ed47da 100644
--- a/src/main/java/org/apache/sysml/runtime/util/PRNGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/util/PRNGenerator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
public abstract class PRNGenerator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/PoissonPRNGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/PoissonPRNGenerator.java b/src/main/java/org/apache/sysml/runtime/util/PoissonPRNGenerator.java
index 1d06442..bc50eb7 100644
--- a/src/main/java/org/apache/sysml/runtime/util/PoissonPRNGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/util/PoissonPRNGenerator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import org.apache.commons.math3.distribution.PoissonDistribution;
import org.apache.commons.math3.random.SynchronizedRandomGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/RandN.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/RandN.java b/src/main/java/org/apache/sysml/runtime/util/RandN.java
index 60c5bf3..70a139c 100644
--- a/src/main/java/org/apache/sysml/runtime/util/RandN.java
+++ b/src/main/java/org/apache/sysml/runtime/util/RandN.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/RandNPair.java b/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
index 11d574f..3c4c36e 100644
--- a/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
+++ b/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
index 07ad879..237086a 100644
--- a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.Arrays;
import java.util.Random;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
/**
* Utilities for sorting, primarily used for SparseRows.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java b/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
index 73eef76..7aedaf0 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index d124ec4..c5a2742 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
public class UtilFunctions
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/BinaryObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/BinaryObject.java b/src/main/java/org/apache/sysml/udf/BinaryObject.java
index c853af0..21c289b 100644
--- a/src/main/java/org/apache/sysml/udf/BinaryObject.java
+++ b/src/main/java/org/apache/sysml/udf/BinaryObject.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
/**
* Class to represent an object.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java b/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java
index 82d2bfe..565f470 100644
--- a/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java
+++ b/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
/**
* Class to maintain external function invocation instructions.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/FunctionParameter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/FunctionParameter.java b/src/main/java/org/apache/sysml/udf/FunctionParameter.java
index 03377ae..4836efc 100644
--- a/src/main/java/org/apache/sysml/udf/FunctionParameter.java
+++ b/src/main/java/org/apache/sysml/udf/FunctionParameter.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/Matrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/Matrix.java b/src/main/java/org/apache/sysml/udf/Matrix.java
index baa5451..d248933 100644
--- a/src/main/java/org/apache/sysml/udf/Matrix.java
+++ b/src/main/java/org/apache/sysml/udf/Matrix.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
import java.io.IOException;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* Class to represent the matrix input type
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/PackageFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/PackageFunction.java b/src/main/java/org/apache/sysml/udf/PackageFunction.java
index 326bc3f..b6f3607 100644
--- a/src/main/java/org/apache/sysml/udf/PackageFunction.java
+++ b/src/main/java/org/apache/sysml/udf/PackageFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
import java.io.Serializable;
import java.util.ArrayList;
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
/**
* Abstract class that should be extended to implement a package function.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java b/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
index a7f7d0a..a5832c0 100644
--- a/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
+++ b/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
/**
* Class to capture a runtime exception during package execution.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/Scalar.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/Scalar.java b/src/main/java/org/apache/sysml/udf/Scalar.java
index 755b9b5..1bf27c6 100644
--- a/src/main/java/org/apache/sysml/udf/Scalar.java
+++ b/src/main/java/org/apache/sysml/udf/Scalar.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf;
+package org.apache.sysml.udf;
/**
* Class to represent a scalar input/output.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java b/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
index 66b2856..6343474 100644
--- a/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Matrix.ValueType;
+import org.apache.sysml.udf.Scalar.ScalarValueType;
/**
* Wrapper class for binning a sorted input vector
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java b/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
index f7fc3b8..593bdd7 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class to deNaN matrices by replacing all NaNs with zeros,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java b/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
index acb6830..73f5d8a 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class to deNegInfinity matrices by replacing all Negative Infinities
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
index c721ae1..aa64ee4 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Matrix.ValueType;
public class DynamicProjectMatrixCP extends PackageFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
index 2df1cbf..2d59921 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
index cdae5a7..d957002 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Matrix.ValueType;
+import org.apache.sysml.udf.Scalar.ScalarValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
index c2b87e8..699e935 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Scalar.ScalarValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java b/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
index 85307d4..6a7124d 100644
--- a/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class for conversions of bit vectors to condensed position vectors.
* The semantics are equivalent to the following dml snippet:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java b/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
index 7b9b448..532761c 100644
--- a/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
import java.util.Arrays;
import java.util.Comparator;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class for Order rows based on values in a column
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java b/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
index 9561d7e..11b4f99 100644
--- a/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
import java.util.Arrays;
import java.util.Comparator;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class for Sorting and Creating of a Permutation Matrix
@@ -43,7 +43,7 @@ import com.ibm.bi.dml.udf.Matrix.ValueType;
*
* permutation_matrix= externalFunction(Matrix[Double] A, Boolean desc) return
* (Matrix[Double] P) implemented in
- * (classname="com.ibm.bi.dml.udf.lib.PermutationMatrixWrapper"
+ * (classname="org.apache.sysml.udf.lib.PermutationMatrixWrapper"
* ,exectype="mem"); A = read( "Data/A.mtx"); P = permutation_matrix( A[,2],
* false); B = P %*% A
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/RemoveEmptyRows.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/RemoveEmptyRows.java b/src/main/java/org/apache/sysml/udf/lib/RemoveEmptyRows.java
index 3d00b7a..32fc218 100644
--- a/src/main/java/org/apache/sysml/udf/lib/RemoveEmptyRows.java
+++ b/src/main/java/org/apache/sysml/udf/lib/RemoveEmptyRows.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java b/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
index 0eb181a..beca10a 100644
--- a/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.udf.lib;
+package org.apache.sysml.udf.lib;
-//import com.ibm.bi.dml.packagesupport.Scalar;
-//import com.ibm.bi.dml.packagesupport.FIO;
+//import org.apache.sysml.packagesupport.Scalar;
+//import org.apache.sysml.packagesupport.FIO;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.Scalar.ScalarValueType;
/**
* Wrapper class for time invocation
*
* time = externalFunction(Integer i) return (Double B) implemented in
- * (classname="com.ibm.bi.dml.udf.lib.TimeWrapper",exectype="mem");
+ * (classname="org.apache.sysml.udf.lib.TimeWrapper",exectype="mem");
*
* t = time (1);
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/AppException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/AppException.java b/src/main/java/org/apache/sysml/utils/AppException.java
index 2ef278f..2f2714b 100644
--- a/src/main/java/org/apache/sysml/utils/AppException.java
+++ b/src/main/java/org/apache/sysml/utils/AppException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
/**
* This exception should be thrown to flag errors by systemML apps
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/Explain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java
index 6b2e7ac..7c2e70f 100644
--- a/src/main/java/org/apache/sysml/utils/Explain.java
+++ b/src/main/java/org/apache/sysml/utils/Explain.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -23,48 +23,48 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFLoopNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CSVReblockSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ReblockSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFLoopNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CSVReblockSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ReblockSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
public class Explain
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/InstallDependencyForIntegrationTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/InstallDependencyForIntegrationTests.java b/src/main/java/org/apache/sysml/utils/InstallDependencyForIntegrationTests.java
index 71814fb..e8a39c9 100644
--- a/src/main/java/org/apache/sysml/utils/InstallDependencyForIntegrationTests.java
+++ b/src/main/java/org/apache/sysml/utils/InstallDependencyForIntegrationTests.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
import static org.junit.Assert.fail;
@@ -23,7 +23,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-//import com.ibm.bi.dml.test.utils.TestUtils;
+//import org.apache.sysml.test.utils.TestUtils;
/**
* This class will be used to setup dependency on Eclipse environment as well as on Jenkins server
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/JSONHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/JSONHelper.java b/src/main/java/org/apache/sysml/utils/JSONHelper.java
index 09e4f57..f986c0b 100644
--- a/src/main/java/org/apache/sysml/utils/JSONHelper.java
+++ b/src/main/java/org/apache/sysml/utils/JSONHelper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
import java.io.IOException;
import java.io.Reader;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/ParameterBuilder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/ParameterBuilder.java b/src/main/java/org/apache/sysml/utils/ParameterBuilder.java
index 66b8762..0da2dc7 100644
--- a/src/main/java/org/apache/sysml/utils/ParameterBuilder.java
+++ b/src/main/java/org/apache/sysml/utils/ParameterBuilder.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
import static org.junit.Assert.fail;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/Statistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java
index b907ad4..b75c1ce 100644
--- a/src/main/java/org/apache/sysml/utils/Statistics.java
+++ b/src/main/java/org/apache/sysml/utils/Statistics.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
import java.lang.management.CompilationMXBean;
import java.lang.management.GarbageCollectorMXBean;
@@ -28,15 +28,15 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
/**
* This class captures all statistics.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/utils/Timer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Timer.java b/src/main/java/org/apache/sysml/utils/Timer.java
index 8fbb40d..b775f9d 100644
--- a/src/main/java/org/apache/sysml/utils/Timer.java
+++ b/src/main/java/org/apache/sysml/utils/Timer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.utils;
+package org.apache.sysml.utils;
public class Timer
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java b/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
index 80670e1..058c2c9 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn;
+package org.apache.sysml.yarn;
import java.io.IOException;
@@ -37,10 +37,10 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLScriptException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLScriptException;
public class DMLAppMaster
{
@@ -54,7 +54,7 @@ public class DMLAppMaster
{
// for internal debugging only
if( DMLYarnClientProxy.LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.yarn").setLevel((Level) Level.DEBUG);
+ Logger.getLogger("org.apache.sysml.yarn").setLevel((Level) Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/DMLAppMasterStatusReporter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMasterStatusReporter.java b/src/main/java/org/apache/sysml/yarn/DMLAppMasterStatusReporter.java
index 35865e3..9eb111b 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLAppMasterStatusReporter.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLAppMasterStatusReporter.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn;
+package org.apache.sysml.yarn;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
index 2390d37..3e3208b 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn;
+package org.apache.sysml.yarn;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,21 +24,21 @@ import java.util.HashMap;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.yarn.ropt.ResourceConfig;
-import com.ibm.bi.dml.yarn.ropt.ResourceOptimizer;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterConfig;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.OptimizerUtils.OptimizationLevel;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.yarn.ropt.ResourceConfig;
+import org.apache.sysml.yarn.ropt.ResourceOptimizer;
+import org.apache.sysml.yarn.ropt.YarnClusterConfig;
public class DMLAppMasterUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java
index 52d6196..17cc27e 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn;
+package org.apache.sysml.yarn;
import java.io.BufferedReader;
import java.io.File;
@@ -52,12 +52,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* NOTES:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/DMLYarnClientProxy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLYarnClientProxy.java b/src/main/java/org/apache/sysml/yarn/DMLYarnClientProxy.java
index 7affc0a..e1b9d2f 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLYarnClientProxy.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLYarnClientProxy.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn;
+package org.apache.sysml.yarn;
import java.io.IOException;
import java.util.ArrayList;
@@ -25,17 +25,17 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.yarn.ropt.ResourceConfig;
-import com.ibm.bi.dml.yarn.ropt.ResourceOptimizer;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterConfig;
-import com.ibm.bi.dml.yarn.ropt.YarnOptimizerUtils;
-import com.ibm.bi.dml.yarn.ropt.YarnOptimizerUtils.GridEnumType;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils.OptimizationLevel;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.yarn.ropt.ResourceConfig;
+import org.apache.sysml.yarn.ropt.ResourceOptimizer;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.yarn.ropt.YarnClusterConfig;
+import org.apache.sysml.yarn.ropt.YarnOptimizerUtils;
+import org.apache.sysml.yarn.ropt.YarnOptimizerUtils.GridEnumType;
/**
* The sole purpose of this class is to serve as a proxy to
@@ -56,7 +56,7 @@ public class DMLYarnClientProxy
{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.yarn")
+ Logger.getLogger("org.apache.sysml.yarn")
.setLevel((Level) Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumeration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumeration.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumeration.java
index 6d9e218..e5cf2d7 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumeration.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumeration.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public abstract class GridEnumeration
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationEqui.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationEqui.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationEqui.java
index e3f8fc2..ed4d0fd 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationEqui.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationEqui.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public class GridEnumerationEqui extends GridEnumeration
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationExp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationExp.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationExp.java
index 506b655..9087ab2 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationExp.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationExp.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public class GridEnumerationExp extends GridEnumeration
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationHybrid.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationHybrid.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationHybrid.java
index 63bea63..3485c2e 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationHybrid.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationHybrid.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
/**
* Composite overlay of hybrid and exp grid.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java
index db18f42..37affed 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
public class GridEnumerationMemory extends GridEnumeration
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/MRJobResourceInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/MRJobResourceInstruction.java b/src/main/java/org/apache/sysml/yarn/ropt/MRJobResourceInstruction.java
index 7e9ea94..b52db89 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/MRJobResourceInstruction.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/MRJobResourceInstruction.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
/**
* The purpose of this class is to encode the yarn mapred memory configuration into
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java
index 7f6f11e..1fc4058 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.util.ArrayList;
import java.util.Collections;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
public class ResourceConfig
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
index c6f6486..5d41aad 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.io.IOException;
import java.util.ArrayList;
@@ -23,42 +23,42 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.cost.CostEstimationWrapper;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTreeConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.yarn.DMLYarnClient;
-import com.ibm.bi.dml.yarn.ropt.YarnOptimizerUtils.GridEnumType;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.cost.CostEstimationWrapper;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptTreeConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.yarn.DMLYarnClient;
+import org.apache.sysml.yarn.ropt.YarnOptimizerUtils.GridEnumType;
/**
* TODO parallel version with exposed numThreads parameter
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
index e1ff5a3..5bf90aa 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
public class ResourceOptimizerCPMigration
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterAnalyzer.java b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterAnalyzer.java
index 4611f57..f3f55c8 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterAnalyzer.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterAnalyzer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,8 +33,8 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRConfigurationNames;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames;
/**
* Central place for analyzing and obtaining static infrastructure properties
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterConfig.java b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterConfig.java
index e6132e7..15aa851 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterConfig.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/YarnClusterConfig.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
public class YarnClusterConfig
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
index a3247bd..5c853e1 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.yarn.ropt;
+package org.apache.sysml.yarn.ropt;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
public class YarnOptimizerUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/standalone/runStandaloneSystemML.bat
----------------------------------------------------------------------
diff --git a/src/main/standalone/runStandaloneSystemML.bat b/src/main/standalone/runStandaloneSystemML.bat
index a425d3a..157e0f4 100644
--- a/src/main/standalone/runStandaloneSystemML.bat
+++ b/src/main/standalone/runStandaloneSystemML.bat
@@ -15,7 +15,7 @@ set LOG4JPROP=log4j.properties
for /f "tokens=1,* delims= " %%a in ("%*") do set ALLBUTFIRST=%%b
-java -Xmx4g -Xms4g -Xmn400m -cp %CLASSPATH% -Dlog4j.configuration=file:%LOG4JPROP% com.ibm.bi.dml.api.DMLScript -f %1 -exec singlenode -config=SystemML-config.xml %ALLBUTFIRST%
+java -Xmx4g -Xms4g -Xmn400m -cp %CLASSPATH% -Dlog4j.configuration=file:%LOG4JPROP% org.apache.sysml.api.DMLScript -f %1 -exec singlenode -config=SystemML-config.xml %ALLBUTFIRST%
GOTO End
:Err
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/standalone/runStandaloneSystemML.sh
----------------------------------------------------------------------
diff --git a/src/main/standalone/runStandaloneSystemML.sh b/src/main/standalone/runStandaloneSystemML.sh
index 98585e7..f20a12d 100644
--- a/src/main/standalone/runStandaloneSystemML.sh
+++ b/src/main/standalone/runStandaloneSystemML.sh
@@ -43,7 +43,7 @@ done
LOG4JPROP=log4j.properties
# invoke the jar with options and arguments
-java -Xmx4g -Xms4g -Xmn400m -cp ${CLASSPATH} -Dlog4j.configuration=file:${LOG4JPROP} com.ibm.bi.dml.api.DMLScript \
+java -Xmx4g -Xms4g -Xmn400m -cp ${CLASSPATH} -Dlog4j.configuration=file:${LOG4JPROP} org.apache.sysml.api.DMLScript \
-f ${SCRIPT_FILE} -exec singlenode -config=$CURRENT_PATH"/SystemML-config.xml" \
$@
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
index dc243f2..5e2e548 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration;
+package org.apache.sysml.test.integration;
import java.util.ArrayList;
import org.junit.After;
import org.junit.Before;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class AutomatedScalabilityTestBase extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 198a325..a4bace6 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration;
+package org.apache.sysml.test.integration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.lops.Lop;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.wink.json4j.JSONObject;
@@ -38,25 +38,25 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.ParameterBuilder;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.ParameterBuilder;
+import org.apache.sysml.utils.Statistics;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
index de12968..681533e 100644
--- a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
+++ b/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration;
+package org.apache.sysml.test.integration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/TestConfiguration.java b/src/test/java/org/apache/sysml/test/integration/TestConfiguration.java
index b2e5889..406537a 100644
--- a/src/test/java/org/apache/sysml/test/integration/TestConfiguration.java
+++ b/src/test/java/org/apache/sysml/test/integration/TestConfiguration.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration;
+package org.apache.sysml.test.integration;
import java.util.HashMap;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
index f7372c1..3dcfaf4 100644
--- a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
+++ b/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration;
+package org.apache.sysml.test.integration;
public class TestMatrixCharacteristics
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java b/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java
index c78a517..1dca12e 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -27,8 +27,8 @@ import java.util.Map;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
public abstract class ApplyTransformTest extends AutomatedTestBase{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
index 5222269..b8120ad 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,10 +25,10 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class ArimaTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java b/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java
index 437c6da..7c45229 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -23,9 +23,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class CsplineCGTest extends AutomatedTestBase {
protected final static String TEST_DIR = "applications/cspline/";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java b/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java
index 8fc3cdf..5158fa5 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class CsplineDSTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
index 884982d..4e332f1 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,10 +26,10 @@ import java.util.Random;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class GLMTest extends AutomatedTestBase
@@ -277,7 +277,7 @@ public abstract class GLMTest extends AutomatedTestBase
}
- int defaultBlockSize = com.ibm.bi.dml.parser.DMLTranslator.DMLBlockSize;
+ int defaultBlockSize = org.apache.sysml.parser.DMLTranslator.DMLBlockSize;
MatrixCharacteristics mc_X = new MatrixCharacteristics (rows, cols, defaultBlockSize, defaultBlockSize, nnz_in_X);
writeInputMatrixWithMTD ("X", X, true, mc_X);
[55/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSquaredLossTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSquaredLossTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSquaredLossTest.java
index b0eabe8..27eb3f4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSquaredLossTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSquaredLossTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedUnaryMatrixMultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedUnaryMatrixMultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedUnaryMatrixMultTest.java
index 078cbd6..085069a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedUnaryMatrixMultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedUnaryMatrixMultTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/BranchRemovalTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/BranchRemovalTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/BranchRemovalTest.java
index d8b61fd..08e21af 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/BranchRemovalTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/BranchRemovalTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class BranchRemovalTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
index 4f5eedb..b9b371e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class CSVReadUnknownSizeTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
index ab159ed..e64d232 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
@@ -23,12 +23,12 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class FunctionRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
index 8e507ba..bfb3aea 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class IPAAssignConstantPropagationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAComplexAppendTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAComplexAppendTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAComplexAppendTest.java
index ee993b3..d26c977 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAComplexAppendTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAComplexAppendTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.io.IOException;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class IPAComplexAppendTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAConstantPropagationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAConstantPropagationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAConstantPropagationTest.java
index 2728215..a7edebf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAConstantPropagationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAConstantPropagationTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class IPAConstantPropagationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAPropagationSizeMultipleFunctionsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAPropagationSizeMultipleFunctionsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAPropagationSizeMultipleFunctionsTest.java
index b1b14ba..1fea1df 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAPropagationSizeMultipleFunctionsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/IPAPropagationSizeMultipleFunctionsTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class IPAPropagationSizeMultipleFunctionsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/LiteralReplaceCastScalarReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/LiteralReplaceCastScalarReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/LiteralReplaceCastScalarReadTest.java
index cfa0f1f..a170fd8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/LiteralReplaceCastScalarReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/LiteralReplaceCastScalarReadTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/MultipleReadsIPATest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/MultipleReadsIPATest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/MultipleReadsIPATest.java
index 9408f8b..40cbcfc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/MultipleReadsIPATest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/MultipleReadsIPATest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class MultipleReadsIPATest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
index 4a7b05b..026fc7c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class PredicateRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandJobRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandJobRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandJobRecompileTest.java
index 60936fa..791a258 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandJobRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandJobRecompileTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class RandJobRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
index ee50c2f..cfb32eb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class RandRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandSizeExpressionEvalTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandSizeExpressionEvalTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandSizeExpressionEvalTest.java
index 19a5071..f1fbfb6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandSizeExpressionEvalTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandSizeExpressionEvalTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class RandSizeExpressionEvalTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
index c542094..af960b3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.io.IOException;
import java.util.HashMap;
@@ -24,16 +24,16 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class ReblockRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
index 8c0db05..184a136 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* The main purpose of this test is to ensure that encountered and fixed
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
index 60a7ff7..3c0afa0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
* INTERESTING NOTE: see MINUS_RIGHT; if '(X+1)-X' instead of '(X+2)-X'
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RewriteComplexMapMultChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RewriteComplexMapMultChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RewriteComplexMapMultChainTest.java
index bfa0239..69e1edc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RewriteComplexMapMultChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RewriteComplexMapMultChainTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.io.IOException;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class RewriteComplexMapMultChainTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
index c0e8625..4ecfa1e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
@@ -15,25 +15,25 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class SparsityFunctionRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
index 8bc3dcc..3d17912 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
@@ -23,19 +23,19 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class SparsityRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/reorg/DiagV2MTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/reorg/DiagV2MTest.java b/src/test/java/org/apache/sysml/test/integration/functions/reorg/DiagV2MTest.java
index abf3f36..107c60f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/reorg/DiagV2MTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/reorg/DiagV2MTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.reorg;
+package org.apache.sysml.test.integration.functions.reorg;
import java.util.HashMap;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class DiagV2MTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullOrderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullOrderTest.java b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullOrderTest.java
index 4f81a2e..4589e71 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullOrderTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullOrderTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.reorg;
+package org.apache.sysml.test.integration.functions.reorg;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* TODO add tests for dynamically computed ordering parameters; in hybrid execution mode
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullTransposeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullTransposeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullTransposeTest.java
index 2fa4c63..a039f8e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullTransposeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/reorg/FullTransposeTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.reorg;
+package org.apache.sysml.test.integration.functions.reorg;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/reorg/MatrixReshapeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/reorg/MatrixReshapeTest.java
index f331ef5..974c69f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/reorg/MatrixReshapeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/reorg/MatrixReshapeTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.reorg;
+package org.apache.sysml.test.integration.functions.reorg;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* NOTE: there are differences to R's matrix operation; in SystemML byrow refers to both input and output
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarMatrixUnaryBinaryTermTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarMatrixUnaryBinaryTermTest.java b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarMatrixUnaryBinaryTermTest.java
index 8bd86dc..d84d8df 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarMatrixUnaryBinaryTermTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarMatrixUnaryBinaryTermTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.terms;
+package org.apache.sysml.test.integration.functions.terms;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ScalarMatrixUnaryBinaryTermTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarToMatrixInLoopTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarToMatrixInLoopTest.java b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarToMatrixInLoopTest.java
index 22d6e56..6e921f7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarToMatrixInLoopTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/terms/ScalarToMatrixInLoopTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.terms;
+package org.apache.sysml.test.integration.functions.terms;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ScalarToMatrixInLoopTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableMatrixIgnoreZerosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableMatrixIgnoreZerosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableMatrixIgnoreZerosTest.java
index 4a896bb..f04ac7b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableMatrixIgnoreZerosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableMatrixIgnoreZerosTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* This test investigates the specific Hop-Lop rewrite for the following pattern:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableSequenceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableSequenceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableSequenceTest.java
index b5cdcfe..051023c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableSequenceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CTableSequenceTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* This test investigates the specific Hop-Lop rewrite ctable(seq(1,nrow(X)),X).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CentralMomentWeightsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CentralMomentWeightsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CentralMomentWeightsTest.java
index f606cd5..4dc2454 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CentralMomentWeightsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CentralMomentWeightsTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CovarianceWeightsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CovarianceWeightsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CovarianceWeightsTest.java
index b66f025..e2177af 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/CovarianceWeightsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/CovarianceWeightsTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/QuantileWeightsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/QuantileWeightsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/QuantileWeightsTest.java
index 4c785f1..788d842 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/QuantileWeightsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/QuantileWeightsTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/ternary/TableOutputTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/ternary/TableOutputTest.java b/src/test/java/org/apache/sysml/test/integration/functions/ternary/TableOutputTest.java
index e582a46..dbfdd4d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/ternary/TableOutputTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/ternary/TableOutputTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class TableOutputTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/transform/RunTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/RunTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/RunTest.java
index 75a1a5b..8b3cb02 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/RunTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/RunTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.transform;
+package org.apache.sysml.test.integration.functions.transform;
import java.io.IOException;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
index 2fd7726..87d4137 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.transform;
+package org.apache.sysml.test.integration.functions.transform;
import static org.junit.Assert.assertTrue;
@@ -29,19 +29,19 @@ import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONObject;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.io.ReaderBinaryBlock;
-import com.ibm.bi.dml.runtime.io.ReaderTextCSV;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.transform.TransformationAgent.TX_METHOD;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.ReaderBinaryBlock;
+import org.apache.sysml.runtime.io.ReaderTextCSV;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.transform.TransformationAgent.TX_METHOD;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
index 9c33fca..1772e6b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.transform;
+package org.apache.sysml.test.integration.functions.transform;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.io.ReaderBinaryBlock;
-import com.ibm.bi.dml.runtime.io.ReaderTextCSV;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.io.ReaderBinaryBlock;
+import org.apache.sysml.runtime.io.ReaderTextCSV;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class TransformTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ACosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ACosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ACosTest.java
index 42353b3..fe1c606 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ACosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ACosTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ASinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ASinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ASinTest.java
index c4fbbc0..655e341 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ASinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ASinTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ATanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ATanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ATanTest.java
index 908026a..c6b5f9e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ATanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ATanTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/AbsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/AbsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/AbsTest.java
index 055afbf..3c4d823 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/AbsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/AbsTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CastAsScalarTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CastAsScalarTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CastAsScalarTest.java
index a36930b..ffe79fc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CastAsScalarTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CastAsScalarTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CosTest.java
index 06c21cf..7a42a35 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CosTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/DiagTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/DiagTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/DiagTest.java
index a2368f0..e6b4264 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/DiagTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/DiagTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/EigenFactorizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/EigenFactorizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/EigenFactorizeTest.java
index 75d2e57..430a041 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/EigenFactorizeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/EigenFactorizeTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class EigenFactorizeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCummaxTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCummaxTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCummaxTest.java
index b0d9572..90ee7fb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCummaxTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCummaxTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumminTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumminTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumminTest.java
index 2456974..a435a34 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumminTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumminTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumprodTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumprodTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumprodTest.java
index 2bc93b4..3dcd9e3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumprodTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumprodTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumsumTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumsumTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumsumTest.java
index 7d5f633..2cd1ef7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumsumTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullCumsumTest.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullSelectPosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullSelectPosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullSelectPosTest.java
index 0c10938..f582eff 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullSelectPosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/FullSelectPosTest.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/IQMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/IQMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/IQMTest.java
index 9dc03e9..216005b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/IQMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/IQMTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/LUFactorizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/LUFactorizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/LUFactorizeTest.java
index 31680ad..720f435 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/LUFactorizeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/LUFactorizeTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class LUFactorizeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MLUnaryBuiltinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MLUnaryBuiltinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MLUnaryBuiltinTest.java
index 0113420..a2b4b9f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MLUnaryBuiltinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MLUnaryBuiltinTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MatrixInverseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MatrixInverseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MatrixInverseTest.java
index 1ff8df7..d435ea5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MatrixInverseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MatrixInverseTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class MatrixInverseTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MinusTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MinusTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MinusTest.java
index 62d6dea..aeb75e1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MinusTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/MinusTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class MinusTest extends AutomatedTestBase
{
[56/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/ScalarIOTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/ScalarIOTest.java
index 79bc0a9..e86bbfa 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/ScalarIOTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/ScalarIOTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ScalarIOTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
index a685038..db0929f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import java.io.IOException;
@@ -23,17 +23,17 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class SeqParReadTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/binary/SerializeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/binary/SerializeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/binary/SerializeTest.java
index b37afda..83381f5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/binary/SerializeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/binary/SerializeTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.binary;
+package org.apache.sysml.test.integration.functions.io.binary;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class SerializeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/CSVParametersTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/CSVParametersTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/CSVParametersTest.java
index 92eb203..f6f5f03 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/CSVParametersTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/CSVParametersTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.csv;
+package org.apache.sysml.test.integration.functions.io.csv;
import java.util.Arrays;
import java.util.Collection;
@@ -25,11 +25,11 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
@RunWith(value = Parameterized.class)
public class CSVParametersTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/FormatChangeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/FormatChangeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/FormatChangeTest.java
index f12675d..12d9564 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/FormatChangeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/FormatChangeTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.csv;
+package org.apache.sysml.test.integration.functions.io.csv;
import java.util.Arrays;
import java.util.Collection;
@@ -26,11 +26,11 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
@RunWith(value = Parameterized.class)
public class FormatChangeTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
index d0ff86a..4124af5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.csv;
+package org.apache.sysml.test.integration.functions.io.csv;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* JUnit Test cases to evaluate the functionality of reading CSV files.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/WriteCSVTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/WriteCSVTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/WriteCSVTest.java
index 5e17dcd..3edc9c7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/WriteCSVTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/WriteCSVTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.csv;
+package org.apache.sysml.test.integration.functions.io.csv;
import java.io.IOException;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* JUnit Test cases to evaluate the functionality of reading CSV files.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/FormatChangeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/FormatChangeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/FormatChangeTest.java
index 66ea65d..d38f620 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/FormatChangeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/FormatChangeTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.matrixmarket;
+package org.apache.sysml.test.integration.functions.io.matrixmarket;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FormatChangeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
index 1888041..18d3518 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.matrixmarket;
+package org.apache.sysml.test.integration.functions.io.matrixmarket;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ReadMMTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/SystemTMulticlassSVMScoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/SystemTMulticlassSVMScoreTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/SystemTMulticlassSVMScoreTest.java
index 8af9d83..86430f8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/SystemTMulticlassSVMScoreTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/SystemTMulticlassSVMScoreTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.jmlc;
+package org.apache.sysml.test.integration.functions.jmlc;
import java.io.IOException;
import java.util.ArrayList;
@@ -23,18 +23,18 @@ import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.jmlc.Connection;
-import com.ibm.bi.dml.api.jmlc.PreparedScript;
-import com.ibm.bi.dml.api.jmlc.ResultVariables;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.jmlc.Connection;
+import org.apache.sysml.api.jmlc.PreparedScript;
+import org.apache.sysml.api.jmlc.ResultVariables;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ConditionalValidateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ConditionalValidateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ConditionalValidateTest.java
index 80ceaec..d77c8b5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ConditionalValidateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ConditionalValidateTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeCastingTest.java
index 6dfd1e0..b31b665 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeCastingTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
index 0329776..5acb072 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.io.BufferedReader;
import java.io.FileReader;
@@ -24,16 +24,16 @@ import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
* GENERAL NOTE
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionInliningTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionInliningTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionInliningTest.java
index 50aef57..7ff2360 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionInliningTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionInliningTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class FunctionInliningTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPALiteralReplacementTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPALiteralReplacementTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPALiteralReplacementTest.java
index 3ab7918..638ed8e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPALiteralReplacementTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPALiteralReplacementTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Regression test for function recompile-once issue with literal replacement.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAScalarRecursionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAScalarRecursionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAScalarRecursionTest.java
index 9f8f155..bf75e5d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAScalarRecursionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAScalarRecursionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAUnknownRecursionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAUnknownRecursionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAUnknownRecursionTest.java
index 2544527..4118659 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAUnknownRecursionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/IPAUnknownRecursionTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class IPAUnknownRecursionTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/InvalidFunctionSignatureTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/InvalidFunctionSignatureTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/InvalidFunctionSignatureTest.java
index 11710c9..4c4b827 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/InvalidFunctionSignatureTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/InvalidFunctionSignatureTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/LongOverflowTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/LongOverflowTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/LongOverflowTest.java
index 9b25916..e41ca12 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/LongOverflowTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/LongOverflowTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolStringTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolStringTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolStringTest.java
index 1b6f926..e3efbd4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolStringTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolStringTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolUnknownCSVReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolUnknownCSVReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolUnknownCSVReadTest.java
index 5b783ed..c09df9f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolUnknownCSVReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/NrowNcolUnknownCSVReadTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* This test checks for valid meta information after csv read with unknown size
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java
index 84f5d11..56bc616 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintExpressionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintExpressionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintExpressionTest.java
index 544fabb..df532a8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintExpressionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintExpressionTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintMatrixTest.java
index ccf8403..68eaa13 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/PrintMatrixTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ReadAfterWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ReadAfterWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ReadAfterWriteTest.java
index 76b85ac..dd25ec2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ReadAfterWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ReadAfterWriteTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSimplifyRowColSumMVMultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSimplifyRowColSumMVMultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSimplifyRowColSumMVMultTest.java
index 4f085f6..3444dc5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSimplifyRowColSumMVMultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSimplifyRowColSumMVMultTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
* Regression test for function recompile-once issue with literal replacement.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSlicedMatrixMultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSlicedMatrixMultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSlicedMatrixMultTest.java
index 7b10e35..2427f62 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSlicedMatrixMultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteSlicedMatrixMultTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarAssignmentTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarAssignmentTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarAssignmentTest.java
index 8a918f5..343905f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarAssignmentTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarAssignmentTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ScalarAssignmentTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarFunctionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarFunctionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarFunctionTest.java
index c75ec0a..5de0b37 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarFunctionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ScalarFunctionTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/SetWorkingDirTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/SetWorkingDirTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/SetWorkingDirTest.java
index d17b8fb..dead06b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/SetWorkingDirTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/SetWorkingDirTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import java.io.File;
import java.io.IOException;
@@ -23,10 +23,10 @@ import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeAutoCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeAutoCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeAutoCastingTest.java
index 96924c2..881c1ba 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeAutoCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeAutoCastingTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeCastingTest.java
index d197fcc..115cd25 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeCastingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ValueTypeCastingTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
index 4c6dd50..bd50efa 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.test.integration.functions.mlcontext;
+package org.apache.sysml.test.integration.functions.mlcontext;
import java.io.IOException;
import java.util.ArrayList;
@@ -16,21 +16,21 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLOutput;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
import org.apache.spark.mllib.linalg.distributed.MatrixEntry;
import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ForLoopPredicateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ForLoopPredicateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ForLoopPredicateTest.java
index fafdce9..1fdf632 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ForLoopPredicateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ForLoopPredicateTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class ForLoopPredicateTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForAdversarialLiteralsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForAdversarialLiteralsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForAdversarialLiteralsTest.java
index d793250..fd25d61 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForAdversarialLiteralsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForAdversarialLiteralsTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForAdversarialLiteralsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForColwiseDataPartitioningTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForColwiseDataPartitioningTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForColwiseDataPartitioningTest.java
index fc2dc86..f98bd95 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForColwiseDataPartitioningTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForColwiseDataPartitioningTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForColwiseDataPartitioningTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDataPartitionLeftIndexingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDataPartitionLeftIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDataPartitionLeftIndexingTest.java
index e5ee1d7..0847739 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDataPartitionLeftIndexingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDataPartitionLeftIndexingTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForDataPartitionLeftIndexingTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
index c6f04aa..e479163 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.io.BufferedReader;
import java.io.FileReader;
@@ -24,15 +24,15 @@ import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
* Different test cases for ParFOR loop dependency analysis:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForFunctionSerializationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForFunctionSerializationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForFunctionSerializationTest.java
index 00222f8..38a8407 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForFunctionSerializationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForFunctionSerializationTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForFunctionSerializationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForMultipleDataPartitioningTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForMultipleDataPartitioningTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForMultipleDataPartitioningTest.java
index 7227986..f1169d2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForMultipleDataPartitioningTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForMultipleDataPartitioningTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForMultipleDataPartitioningTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForNaNResultMergeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForNaNResultMergeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForNaNResultMergeTest.java
index fa89235..267cb58 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForNaNResultMergeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForNaNResultMergeTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import java.util.Map.Entry;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForNaNResultMergeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForParallelRemoteResultMergeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForParallelRemoteResultMergeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForParallelRemoteResultMergeTest.java
index be49f9d..48a2b4a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForParallelRemoteResultMergeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForParallelRemoteResultMergeTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
@@ -23,11 +23,11 @@ import org.junit.Assert;
import org.junit.Test;
import org.junit.Ignore;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class ParForParallelRemoteResultMergeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRepeatedOptimizationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRepeatedOptimizationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRepeatedOptimizationTest.java
index 4427d7f..9b7fddb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRepeatedOptimizationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRepeatedOptimizationTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
@@ -23,16 +23,16 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class ParForRepeatedOptimizationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
index 06da6e1..41204e0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
@@ -23,11 +23,11 @@ import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForReplaceThreadIDRecompileTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRowwiseDataPartitioningTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRowwiseDataPartitioningTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRowwiseDataPartitioningTest.java
index 4b705aa..b4beaae 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRowwiseDataPartitioningTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRowwiseDataPartitioningTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForRowwiseDataPartitioningTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRulebasedOptimizerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRulebasedOptimizerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRulebasedOptimizerTest.java
index 99c9ff2..a07e979 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRulebasedOptimizerTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForRulebasedOptimizerTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForRulebasedOptimizerTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForSerialRemoteResultMergeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForSerialRemoteResultMergeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForSerialRemoteResultMergeTest.java
index 009f8d0..dd8698e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForSerialRemoteResultMergeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForSerialRemoteResultMergeTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class ParForSerialRemoteResultMergeTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest1.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest1.java b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest1.java
index a2e97e9..1c6a247 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest1.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest1.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.piggybacking;
+package org.apache.sysml.test.integration.functions.piggybacking;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class PiggybackingTest1 extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest2.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest2.java
index c8399bc..da185c1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest2.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/piggybacking/PiggybackingTest2.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.piggybacking;
+package org.apache.sysml.test.integration.functions.piggybacking;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class PiggybackingTest2 extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/RewritesWeightedSigmoidTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/RewritesWeightedSigmoidTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/RewritesWeightedSigmoidTest.java
index 87647bc..e2c6dd7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/RewritesWeightedSigmoidTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/RewritesWeightedSigmoidTest.java
@@ -15,25 +15,25 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedCrossEntropyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedCrossEntropyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedCrossEntropyTest.java
index 757c29f..c0b095b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedCrossEntropyTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedCrossEntropyTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedCrossEntropyR;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
index c20f7f6..64ee9fc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedDivMatrixMultTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSigmoidTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSigmoidTest.java b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSigmoidTest.java
index 814f1b0..b1cf5be 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSigmoidTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/quaternary/WeightedSigmoidTest.java
@@ -15,25 +15,25 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
[68/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
index 717fd6b..cafd774 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.io.MatrixWriterFactory;
-import com.ibm.bi.dml.runtime.io.WriterBinaryBlock;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.io.MatrixWriterFactory;
+import org.apache.sysml.runtime.io.WriterBinaryBlock;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/DoubleObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DoubleObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DoubleObject.java
index 70b4a68..3df21d8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DoubleObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DoubleObject.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class DoubleObject extends ScalarObject
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileCPInstruction.java
index 7a631db..c0708da 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileCPInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.RemoveFile;
-import com.ibm.bi.dml.runtime.functionobjects.RenameFile;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.RemoveFile;
+import org.apache.sysml.runtime.functionobjects.RenameFile;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.MapReduceTool;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
index 158c339..7a469e9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class FileObject extends Data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
index b5d1aa2..a3812c8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
import java.util.Collection;
@@ -23,21 +23,21 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/IntObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/IntObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/IntObject.java
index 3bcda0c..f850cf6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/IntObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/IntObject.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Expression.ValueType;
public class IntObject extends ScalarObject
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/KahanObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/KahanObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/KahanObject.java
index 9ee12f5..fca725b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/KahanObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/KahanObject.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class KahanObject extends Data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
index f8b77ef..8f9b1b8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMTSJCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMTSJCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMTSJCPInstruction.java
index e9ab822..078b568 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMTSJCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMTSJCPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
index face2db..a2f323f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.LibCommonsMath;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public class MatrixBuiltinCPInstruction extends BuiltinUnaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
index 9cedcda..1d6a0bc 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.IndexRange;
public class MatrixIndexingCPInstruction extends UnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixArithmeticCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixArithmeticCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixArithmeticCPInstruction.java
index e262dff..41bda85 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixArithmeticCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixArithmeticCPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixArithmeticCPInstruction extends ArithmeticBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixBuiltinCPInstruction.java
index 2e452f5..e15770d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixBuiltinCPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.LibCommonsMath;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixBuiltinCPInstruction extends BuiltinBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixRelationalCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixRelationalCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixRelationalCPInstruction.java
index f19437f..3c90a14 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixRelationalCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixMatrixRelationalCPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixRelationalCPInstruction extends RelationalBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixReshapeCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixReshapeCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixReshapeCPInstruction.java
index bb658df..116baff 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixReshapeCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixReshapeCPInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixScalarBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixScalarBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixScalarBuiltinCPInstruction.java
index 648df9c..a48c0d6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixScalarBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixScalarBuiltinCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
public class MatrixScalarBuiltinCPInstruction extends BuiltinBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java
index 96c1f50..b136062 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.LibCommonsMath;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MultiReturnBuiltinCPInstruction extends ComputationCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/PMMJCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/PMMJCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/PMMJCPInstruction.java
index 0e44bd3..bfb0cbc 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/PMMJCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/PMMJCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index 1b14d05..81e60b4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.IOException;
import java.util.HashMap;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.transform.DataTransform;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.ParameterizedBuiltin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.transform.DataTransform;
public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
index dfd1fc9..8472062 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.IOException;
-import com.ibm.bi.dml.lops.PickByCount.OperationTypes;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.PickByCount.OperationTypes;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class QuantilePickCPInstruction extends BinaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantileSortCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantileSortCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantileSortCPInstruction.java
index e9ec52d..d4dd5cb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantileSortCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantileSortCPInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public class QuantileSortCPInstruction extends UnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
index 42889fc..cc82693 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/RelationalBinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/RelationalBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/RelationalBinaryCPInstruction.java
index bb83c6f..720abf3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/RelationalBinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/RelationalBinaryCPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class RelationalBinaryCPInstruction extends BinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ReorgCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ReorgCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ReorgCPInstruction.java
index 328bd88..5e8d40d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ReorgCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ReorgCPInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SortIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.SortIndex;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class ReorgCPInstruction extends UnaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
index da5b04a..37f4e8d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public class ScalarBuiltinCPInstruction extends BuiltinUnaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixArithmeticCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixArithmeticCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixArithmeticCPInstruction.java
index fbe5bcf..528d757 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixArithmeticCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixArithmeticCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
// TODO rename to MatrixScalar...
public class ScalarMatrixArithmeticCPInstruction extends ArithmeticBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixRelationalCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixRelationalCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixRelationalCPInstruction.java
index 2f19f10..4f89744 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixRelationalCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarMatrixRelationalCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
//TODO rename to MatrixScalar...
public class ScalarMatrixRelationalCPInstruction extends RelationalBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarObject.java
index 8b7b575..11dd551 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarObject.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public abstract class ScalarObject extends Data
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
index cb71894..27badfb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Divide;
-import com.ibm.bi.dml.runtime.functionobjects.Power;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Divide;
+import org.apache.sysml.runtime.functionobjects.Power;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class ScalarScalarArithmeticCPInstruction extends ArithmeticBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
index 774b5d1..9d957e4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class ScalarScalarBuiltinCPInstruction extends BuiltinBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarRelationalCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarRelationalCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarRelationalCPInstruction.java
index 5e295f6..8d45723 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarRelationalCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarRelationalCPInstruction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class ScalarScalarRelationalCPInstruction extends RelationalBinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringInitCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/StringInitCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringInitCPInstruction.java
index ba94eda..5aa9182 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/StringInitCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringInitCPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.util.StringTokenizer;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class StringInitCPInstruction extends UnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/StringObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringObject.java
index 49afe84..a93db7a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/StringObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/StringObject.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
public class StringObject extends ScalarObject
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/TernaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/TernaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/TernaryCPInstruction.java
index 47c293c..b9274b7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/TernaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/TernaryCPInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.Ternary;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.lops.Ternary;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.DataConverter;
public class TernaryCPInstruction extends ComputationCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
index 411e89a..1d47813 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixOuterAgg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.lops.UAggOuterChain;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibMatrixOuterAgg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class UaggOuterChainCPInstruction extends UnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/UnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/UnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/UnaryCPInstruction.java
index 142dbf8..8257138 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/UnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/UnaryCPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.Not;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.Not;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public abstract class UnaryCPInstruction extends ComputationCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 7d19c9d..7e02b49 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -15,36 +15,36 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.io.WriterMatrixMarket;
-import com.ibm.bi.dml.runtime.io.WriterTextCSV;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.io.WriterMatrixMarket;
+import org.apache.sysml.runtime.io.WriterTextCSV;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class VariableCPInstruction extends CPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
index ce8b952..67a7882 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cpfile;
+package org.apache.sysml.runtime.instructions.cpfile;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.MatrixIndexingCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.MatrixIndexingCPInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* This instruction is used if a single partition is too large to fit in memory.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
index 5854b10..18ae41c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cpfile;
+package org.apache.sysml.runtime.instructions.cpfile;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -40,38 +40,38 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.Cell;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.functionobjects.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.util.Cell;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
+import org.apache.sysml.runtime.functionobjects.ParameterizedBuiltin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixWriter;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* File-based (out-of-core) realization of remove empty for robustness because there is no
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateBinaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateBinaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateBinaryInstruction.java
index f982fee..a644e67 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateBinaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateBinaryInstruction.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.MMCJ.MMCJType;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMult.CacheType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MMCJ.MMCJType;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.MapMult.CacheType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AggregateBinaryInstruction extends BinaryMRInstructionBase implements IDistributedCacheConsumer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateInstruction.java
index c5c83a5..b16dcca 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AggregateInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateUnaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateUnaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateUnaryInstruction.java
index 7bb56e4..f9ecb9c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateUnaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AggregateUnaryInstruction.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceDiag;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.ReduceDiag;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AggregateUnaryInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendGInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendGInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendGInstruction.java
index b160ab8..d7e8444 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendGInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendGInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AppendGInstruction extends AppendInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendInstruction.java
index 41476ee..fde4ae5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AppendInstruction extends BinaryMRInstructionBase
[30/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteForLoopVectorization.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteForLoopVectorization.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteForLoopVectorization.java
deleted file mode 100644
index 49af864..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteForLoopVectorization.java
+++ /dev/null
@@ -1,398 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: Simplify program structure by pulling if or else statement body out
- * (removing the if statement block ifself) in order to allow intra-procedure
- * analysis to propagate exact statistics.
- *
- */
-public class RewriteForLoopVectorization extends StatementBlockRewriteRule
-{
-
- private static final OpOp2[] MAP_SCALAR_AGGREGATE_SOURCE_OPS = new OpOp2[]{OpOp2.PLUS, OpOp2.MULT, OpOp2.MIN, OpOp2.MAX};
- private static final AggOp[] MAP_SCALAR_AGGREGATE_TARGET_OPS = new AggOp[]{AggOp.SUM, AggOp.PROD, AggOp.MIN, AggOp.MAX};
-
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- if( sb instanceof ForStatementBlock )
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fs = (ForStatement) fsb.getStatement(0);
- Hop from = fsb.getFromHops();
- Hop to = fsb.getToHops();
- Hop incr = fsb.getIncrementHops();
- String iterVar = fsb.getIterPredicate().getIterVar().getName();
-
- if( fs.getBody()!=null && fs.getBody().size()==1 ) //single child block
- {
- StatementBlock csb = (StatementBlock) fs.getBody().get(0);
- if( !( csb instanceof WhileStatementBlock //last level block
- || csb instanceof IfStatementBlock
- || csb instanceof ForStatementBlock ) )
- {
- //auto vectorzation pattern
- sb = vectorizeScalarAggregate(sb, csb, from, to, incr, iterVar); //e.g., for(i){s = s + as.scalar(X[i,2])}
- sb = vectorizeElementwiseBinary(sb, csb, from, to, incr, iterVar);
- sb = vectorizeElementwiseUnary(sb, csb, from, to, incr, iterVar);
- }
- }
- }
-
- //if no rewrite applied sb is the original for loop otherwise a last level statement block
- //that includes the equivalent vectorized operations.
- ret.add( sb );
-
- return ret;
- }
-
- /**
- * Note: unnecessary row or column indexing then later removed via
- * dynamic rewrites
- *
- * @param sb
- * @param csb
- * @param from
- * @param to
- * @param increment
- * @param itervar
- * @return
- * @throws HopsException
- */
- private StatementBlock vectorizeScalarAggregate( StatementBlock sb, StatementBlock csb, Hop from, Hop to, Hop increment, String itervar )
- throws HopsException
- {
- StatementBlock ret = sb;
-
- //check supported increment values
- if( !(increment instanceof LiteralOp && ((LiteralOp)increment).getDoubleValue()==1.0) ){
- return ret;
- }
-
- //check for applicability
- boolean leftScalar = false;
- boolean rightScalar = false;
- boolean rowIx = false; //row or col
-
- if( csb.get_hops()!=null && csb.get_hops().size()==1 ){
- Hop root = csb.get_hops().get(0);
-
- if( root.getDataType()==DataType.SCALAR && root.getInput().get(0) instanceof BinaryOp ) {
- BinaryOp bop = (BinaryOp) root.getInput().get(0);
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- //check for left scalar plus
- if( HopRewriteUtils.isValidOp(bop.getOp(), MAP_SCALAR_AGGREGATE_SOURCE_OPS)
- && left instanceof DataOp && left.getDataType() == DataType.SCALAR
- && root.getName().equals(left.getName())
- && right instanceof UnaryOp && ((UnaryOp) right).getOp() == OpOp1.CAST_AS_SCALAR
- && right.getInput().get(0) instanceof IndexingOp )
- {
- IndexingOp ix = (IndexingOp)right.getInput().get(0);
- if( ix.getRowLowerEqualsUpper() && ix.getInput().get(1) instanceof DataOp
- && ix.getInput().get(1).getName().equals(itervar) ){
- leftScalar = true;
- rowIx = true;
- }
- else if( ix.getColLowerEqualsUpper() && ix.getInput().get(3) instanceof DataOp
- && ix.getInput().get(3).getName().equals(itervar) ){
- leftScalar = true;
- rowIx = false;
- }
- }
- //check for right scalar plus
- else if( HopRewriteUtils.isValidOp(bop.getOp(), MAP_SCALAR_AGGREGATE_SOURCE_OPS)
- && right instanceof DataOp && right.getDataType() == DataType.SCALAR
- && root.getName().equals(right.getName())
- && left instanceof UnaryOp && ((UnaryOp) left).getOp() == OpOp1.CAST_AS_SCALAR
- && left.getInput().get(0) instanceof IndexingOp )
- {
- IndexingOp ix = (IndexingOp)left.getInput().get(0);
- if( ix.getRowLowerEqualsUpper() && ix.getInput().get(1) instanceof DataOp
- && ix.getInput().get(1).getName().equals(itervar) ){
- rightScalar = true;
- rowIx = true;
- }
- else if( ix.getColLowerEqualsUpper() && ix.getInput().get(3) instanceof DataOp
- && ix.getInput().get(3).getName().equals(itervar) ){
- rightScalar = true;
- rowIx = false;
- }
- }
- }
- }
-
- //apply rewrite if possible
- if( leftScalar || rightScalar )
- {
- Hop root = csb.get_hops().get(0);
- BinaryOp bop = (BinaryOp) root.getInput().get(0);
- Hop cast = bop.getInput().get( leftScalar?1:0 );
- Hop ix = cast.getInput().get(0);
- int aggOpPos = HopRewriteUtils.getValidOpPos(bop.getOp(), MAP_SCALAR_AGGREGATE_SOURCE_OPS);
- AggOp aggOp = MAP_SCALAR_AGGREGATE_TARGET_OPS[aggOpPos];
- //replace cast with sum
- AggUnaryOp newSum = new AggUnaryOp(cast.getName(), DataType.SCALAR, ValueType.DOUBLE, aggOp, Direction.RowCol, ix);
- HopRewriteUtils.removeChildReference(cast, ix);
- HopRewriteUtils.removeChildReference(bop, cast);
- HopRewriteUtils.addChildReference(bop, newSum, leftScalar?1:0 );
- //modify indexing expression according to loop predicate from-to
- //NOTE: any redundant index operations are removed via dynamic algebraic simplification rewrites
- int index1 = rowIx ? 1 : 3;
- int index2 = rowIx ? 2 : 4;
- HopRewriteUtils.removeChildReferenceByPos(ix, ix.getInput().get(index1), index1);
- HopRewriteUtils.addChildReference(ix, from, index1);
- HopRewriteUtils.removeChildReferenceByPos(ix, ix.getInput().get(index2), index2);
- HopRewriteUtils.addChildReference(ix, to, index2);
-
- ret = csb;
- //ret.liveIn().removeVariable(itervar);
- LOG.debug("Applied vectorizeScalarSumForLoop.");
- }
-
- return ret;
- }
-
- /**
- * Note: unnecessary row or column indexing then later removed via
- * dynamic rewrites
- *
- * @param sb
- * @param csb
- * @param from
- * @param to
- * @param increment
- * @param itervar
- * @return
- * @throws HopsException
- */
- private StatementBlock vectorizeElementwiseBinary( StatementBlock sb, StatementBlock csb, Hop from, Hop to, Hop increment, String itervar )
- throws HopsException
- {
- StatementBlock ret = sb;
-
- //check supported increment values
- if( !(increment instanceof LiteralOp && ((LiteralOp)increment).getDoubleValue()==1.0) ){
- return ret;
- }
-
- //check for applicability
- boolean apply = false;
- boolean rowIx = false; //row or col
- if( csb.get_hops()!=null && csb.get_hops().size()==1 )
- {
- Hop root = csb.get_hops().get(0);
-
- if( root.getDataType()==DataType.MATRIX && root.getInput().get(0) instanceof LeftIndexingOp )
- {
- LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
- Hop lixlhs = lix.getInput().get(0);
- Hop lixrhs = lix.getInput().get(1);
-
- if( lixlhs instanceof DataOp && lixrhs instanceof BinaryOp
- && lixrhs.getInput().get(0) instanceof IndexingOp
- && lixrhs.getInput().get(1) instanceof IndexingOp
- && lixrhs.getInput().get(0).getInput().get(0) instanceof DataOp
- && lixrhs.getInput().get(1).getInput().get(0) instanceof DataOp)
- {
- IndexingOp rix0 = (IndexingOp) lixrhs.getInput().get(0);
- IndexingOp rix1 = (IndexingOp) lixrhs.getInput().get(1);
-
- //check for rowwise
- if( lix.getRowLowerEqualsUpper() && rix0.getRowLowerEqualsUpper() && rix1.getRowLowerEqualsUpper()
- && lix.getInput().get(2).getName().equals(itervar)
- && rix0.getInput().get(1).getName().equals(itervar)
- && rix1.getInput().get(1).getName().equals(itervar))
- {
- apply = true;
- rowIx = true;
- }
- //check for colwise
- if( lix.getColLowerEqualsUpper() && rix0.getColLowerEqualsUpper() && rix1.getColLowerEqualsUpper()
- && lix.getInput().get(4).getName().equals(itervar)
- && rix0.getInput().get(3).getName().equals(itervar)
- && rix1.getInput().get(3).getName().equals(itervar))
- {
- apply = true;
- rowIx = false;
- }
- }
- }
- }
-
- //apply rewrite if possible
- if( apply )
- {
- Hop root = csb.get_hops().get(0);
- LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
- BinaryOp bop = (BinaryOp) lix.getInput().get(1);
- IndexingOp rix0 = (IndexingOp) bop.getInput().get(0);
- IndexingOp rix1 = (IndexingOp) bop.getInput().get(1);
- int index1 = rowIx ? 2 : 4;
- int index2 = rowIx ? 3 : 5;
- //modify left indexing bounds
- HopRewriteUtils.removeChildReferenceByPos(lix, lix.getInput().get(index1), index1 );
- HopRewriteUtils.addChildReference(lix, from, index1);
- HopRewriteUtils.removeChildReferenceByPos(lix, lix.getInput().get(index2), index2 );
- HopRewriteUtils.addChildReference(lix, to, index2);
- //modify both right indexing
- HopRewriteUtils.removeChildReferenceByPos(rix0, rix0.getInput().get(index1-1), index1-1 );
- HopRewriteUtils.addChildReference(rix0, from, index1-1);
- HopRewriteUtils.removeChildReferenceByPos(rix0, rix0.getInput().get(index2-1), index2-1 );
- HopRewriteUtils.addChildReference(rix0, to, index2-1);
- HopRewriteUtils.removeChildReferenceByPos(rix1, rix1.getInput().get(index1-1), index1-1 );
- HopRewriteUtils.addChildReference(rix1, from, index1-1);
- HopRewriteUtils.removeChildReferenceByPos(rix1, rix1.getInput().get(index2-1), index2-1 );
- HopRewriteUtils.addChildReference(rix1, to, index2-1);
- rix0.refreshSizeInformation();
- rix1.refreshSizeInformation();
- bop.refreshSizeInformation();
- lix.refreshSizeInformation();
-
- ret = csb;
- //ret.liveIn().removeVariable(itervar);
- LOG.debug("Applied vectorizeElementwiseBinaryForLoop.");
- }
-
- return ret;
- }
-
- /**
- * Note: unnecessary row or column indexing then later removed via
- * dynamic rewrites
- *
- * @param sb
- * @param csb
- * @param from
- * @param to
- * @param increment
- * @param itervar
- * @return
- * @throws HopsException
- */
- private StatementBlock vectorizeElementwiseUnary( StatementBlock sb, StatementBlock csb, Hop from, Hop to, Hop increment, String itervar )
- throws HopsException
- {
- StatementBlock ret = sb;
-
- //check supported increment values
- if( !(increment instanceof LiteralOp && ((LiteralOp)increment).getDoubleValue()==1.0) ){
- return ret;
- }
-
- //check for applicability
- boolean apply = false;
- boolean rowIx = false; //row or col
- if( csb.get_hops()!=null && csb.get_hops().size()==1 )
- {
- Hop root = csb.get_hops().get(0);
-
- if( root.getDataType()==DataType.MATRIX && root.getInput().get(0) instanceof LeftIndexingOp )
- {
- LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
- Hop lixlhs = lix.getInput().get(0);
- Hop lixrhs = lix.getInput().get(1);
-
- if( lixlhs instanceof DataOp && lixrhs instanceof UnaryOp
- && lixrhs.getInput().get(0) instanceof IndexingOp
- && lixrhs.getInput().get(0).getInput().get(0) instanceof DataOp )
- {
- IndexingOp rix = (IndexingOp) lixrhs.getInput().get(0);
- //check for rowwise
- if( lix.getRowLowerEqualsUpper() && rix.getRowLowerEqualsUpper()
- && lix.getInput().get(2).getName().equals(itervar)
- && rix.getInput().get(1).getName().equals(itervar) )
- {
- apply = true;
- rowIx = true;
- }
- //check for colwise
- if( lix.getColLowerEqualsUpper() && rix.getColLowerEqualsUpper()
- && lix.getInput().get(4).getName().equals(itervar)
- && rix.getInput().get(3).getName().equals(itervar) )
- {
- apply = true;
- rowIx = false;
- }
- }
- }
- }
-
- //apply rewrite if possible
- if( apply )
- {
- Hop root = csb.get_hops().get(0);
- LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
- UnaryOp uop = (UnaryOp) lix.getInput().get(1);
- IndexingOp rix = (IndexingOp) uop.getInput().get(0);
- int index1 = rowIx ? 2 : 4;
- int index2 = rowIx ? 3 : 5;
- //modify left indexing bounds
- HopRewriteUtils.removeChildReferenceByPos(lix, lix.getInput().get(index1), index1 );
- HopRewriteUtils.addChildReference(lix, from, index1);
- HopRewriteUtils.removeChildReferenceByPos(lix, lix.getInput().get(index2), index2 );
- HopRewriteUtils.addChildReference(lix, to, index2);
- //modify right indexing
- HopRewriteUtils.removeChildReferenceByPos(rix, rix.getInput().get(index1-1), index1-1 );
- HopRewriteUtils.addChildReference(rix, from, index1-1);
- HopRewriteUtils.removeChildReferenceByPos(rix, rix.getInput().get(index2-1), index2-1 );
- HopRewriteUtils.addChildReference(rix, to, index2-1);
- rix.refreshSizeInformation();
- uop.refreshSizeInformation();
- lix.refreshSizeInformation();
-
- ret = csb;
- //ret.liveIn().removeVariable(itervar);
- LOG.debug("Applied vectorizeElementwiseUnaryForLoop.");
- }
-
- return ret;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteIndexingVectorization.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteIndexingVectorization.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteIndexingVectorization.java
deleted file mode 100644
index 226812d..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteIndexingVectorization.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: Indexing vectorization. This rewrite rule set simplifies
- * multiple right / left indexing accesses within a DAG into row/column
- * index accesses, which is beneficial for two reasons: (1) it is an
- * enabler for later row/column partitioning, and (2) it reduces the number
- * of operations over potentially large data (i.e., prevents unnecessary MR
- * operations and reduces pressure on the buffer pool due to copy on write
- * on left indexing).
- *
- */
-public class RewriteIndexingVectorization extends HopRewriteRule
-{
-
- private static final Log LOG = LogFactory.getLog(RewriteIndexingVectorization.class.getName());
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return roots;
-
- for( Hop h : roots )
- rule_IndexingVectorization( h );
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return root;
-
- rule_IndexingVectorization( root );
-
- return root;
- }
-
-
- /**
- *
- * @param hop
- * @param descendFirst
- * @throws HopsException
- */
- private void rule_IndexingVectorization( Hop hop )
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- //recursively process children
- for( int i=0; i<hop.getInput().size(); i++)
- {
- Hop hi = hop.getInput().get(i);
-
- //apply indexing vectorization rewrites
- //MB: disabled right indexing rewrite because (1) piggybacked in MR anyway, (2) usually
- //not too much overhead, and (3) makes literal replacement more difficult
- //vectorizeRightIndexing( hi ); //e.g., multiple rightindexing X[i,1], X[i,3] -> X[i,];
- vectorizeLeftIndexing( hi ); //e.g., multiple left indexing X[i,1], X[i,3] -> X[i,];
-
- //process childs recursively after rewrites
- rule_IndexingVectorization( hi );
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
- /**
- * Note: unnecessary row or column indexing then later removed via
- * dynamic rewrites
- *
- * @param hop
- * @throws HopsException
- */
- @SuppressWarnings("unused")
- private void vectorizeRightIndexing( Hop hop )
- throws HopsException
- {
- if( hop instanceof IndexingOp ) //right indexing
- {
- IndexingOp ihop0 = (IndexingOp) hop;
- boolean isSingleRow = ihop0.getRowLowerEqualsUpper();
- boolean isSingleCol = ihop0.getColLowerEqualsUpper();
- boolean appliedRow = false;
-
- //search for multiple indexing in same row
- if( isSingleRow && isSingleCol ){
- Hop input = ihop0.getInput().get(0);
- //find candidate set
- //dependence on common subexpression elimination to find equal input / row expression
- ArrayList<Hop> ihops = new ArrayList<Hop>();
- ihops.add(ihop0);
- for( Hop c : input.getParent() ){
- if( c != ihop0 && c instanceof IndexingOp && c.getInput().get(0) == input
- && ((IndexingOp) c).getRowLowerEqualsUpper()
- && c.getInput().get(1)==ihop0.getInput().get(1) )
- {
- ihops.add( c );
- }
- }
- //apply rewrite if found candidates
- if( ihops.size() > 1 ){
- //new row indexing operator
- IndexingOp newRix = new IndexingOp("tmp", DataType.MATRIX, ValueType.DOUBLE, input,
- ihop0.getInput().get(1), ihop0.getInput().get(1), new LiteralOp(1),
- HopRewriteUtils.createValueHop(input, false), true, false);
- HopRewriteUtils.setOutputParameters(newRix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newRix.refreshSizeInformation();
- //rewire current operator and all candidates
- for( Hop c : ihops ) {
- HopRewriteUtils.removeChildReference(c, input); //input data
- HopRewriteUtils.addChildReference(c, newRix, 0);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(1),1); //row lower expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 1);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(2),2); //row upper expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 2);
- c.refreshSizeInformation();
- }
-
- appliedRow = true;
- LOG.debug("Applied vectorizeRightIndexingRow");
- }
- }
-
- //search for multiple indexing in same col
- if( isSingleRow && isSingleCol && !appliedRow ){
- Hop input = ihop0.getInput().get(0);
- //find candidate set
- //dependence on common subexpression elimination to find equal input / row expression
- ArrayList<Hop> ihops = new ArrayList<Hop>();
- ihops.add(ihop0);
- for( Hop c : input.getParent() ){
- if( c != ihop0 && c instanceof IndexingOp && c.getInput().get(0) == input
- && ((IndexingOp) c).getColLowerEqualsUpper()
- && c.getInput().get(3)==ihop0.getInput().get(3) )
- {
- ihops.add( c );
- }
- }
- //apply rewrite if found candidates
- if( ihops.size() > 1 ){
- //new row indexing operator
- IndexingOp newRix = new IndexingOp("tmp", DataType.MATRIX, ValueType.DOUBLE, input,
- new LiteralOp(1), HopRewriteUtils.createValueHop(input, true),
- ihop0.getInput().get(3), ihop0.getInput().get(3), false, true);
- HopRewriteUtils.setOutputParameters(newRix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newRix.refreshSizeInformation();
- //rewire current operator and all candidates
- for( Hop c : ihops ) {
- HopRewriteUtils.removeChildReference(c, input); //input data
- HopRewriteUtils.addChildReference(c, newRix, 0);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(3),3); //col lower expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 3);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(4),4); //col upper expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 4);
- c.refreshSizeInformation();
- }
-
- LOG.debug("Applied vectorizeRightIndexingCol");
- }
- }
- }
- }
-
- /**
- *
- * @param hop
- * @throws HopsException
- */
- @SuppressWarnings("unchecked")
- private void vectorizeLeftIndexing( Hop hop )
- throws HopsException
- {
- if( hop instanceof LeftIndexingOp ) //left indexing
- {
- LeftIndexingOp ihop0 = (LeftIndexingOp) hop;
- boolean isSingleRow = ihop0.getRowLowerEqualsUpper();
- boolean isSingleCol = ihop0.getColLowerEqualsUpper();
- boolean appliedRow = false;
-
- if( isSingleRow && isSingleCol )
- {
- //collect simple chains (w/o multiple consumers) of left indexing ops
- ArrayList<Hop> ihops = new ArrayList<Hop>();
- ihops.add(ihop0);
- Hop current = ihop0;
- while( current.getInput().get(0) instanceof LeftIndexingOp ) {
- LeftIndexingOp tmp = (LeftIndexingOp) current.getInput().get(0);
- if( tmp.getParent().size()>1 //multiple consumers, i.e., not a simple chain
- || !((LeftIndexingOp) tmp).getRowLowerEqualsUpper() //row merge not applicable
- || tmp.getInput().get(2) != ihop0.getInput().get(2) //not the same row
- || tmp.getInput().get(0).getDim2() <= 1 ) //target is single column or unknown
- {
- break;
- }
- ihops.add( tmp );
- current = tmp;
- }
-
- //apply rewrite if found candidates
- if( ihops.size() > 1 ){
- Hop input = current.getInput().get(0);
- Hop rowExpr = ihop0.getInput().get(2); //keep before reset
-
- //new row indexing operator
- IndexingOp newRix = new IndexingOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, input,
- rowExpr, rowExpr, new LiteralOp(1),
- HopRewriteUtils.createValueHop(input, false), true, false);
- HopRewriteUtils.setOutputParameters(newRix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newRix.refreshSizeInformation();
-
- //rewrite bottom left indexing operator
- HopRewriteUtils.removeChildReference(current, input); //input data
- HopRewriteUtils.addChildReference(current, newRix, 0);
-
- //reset row index all candidates
- for( Hop c : ihops ) {
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(2), 2); //row lower expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 2);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(3), 3); //row upper expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 3);
- c.refreshSizeInformation();
- }
-
- //new row left indexing operator (for all parents, only intermediates are guaranteed to have 1 parent)
- //(note: it's important to clone the parent list before creating newLix on top of ihop0)
- ArrayList<Hop> ihop0parents = (ArrayList<Hop>) ihop0.getParent().clone();
- ArrayList<Integer> ihop0parentsPos = new ArrayList<Integer>();
- for( Hop parent : ihop0parents ) {
- int posp = HopRewriteUtils.getChildReferencePos(parent, ihop0);
- HopRewriteUtils.removeChildReferenceByPos(parent, ihop0, posp); //input data
- ihop0parentsPos.add(posp);
- }
-
- LeftIndexingOp newLix = new LeftIndexingOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, input, ihop0,
- rowExpr, rowExpr, new LiteralOp(1),
- HopRewriteUtils.createValueHop(input, false), true, false);
- HopRewriteUtils.setOutputParameters(newLix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newLix.refreshSizeInformation();
-
- for( int i=0; i<ihop0parentsPos.size(); i++ ) {
- Hop parent = ihop0parents.get(i);
- int posp = ihop0parentsPos.get(i);
- HopRewriteUtils.addChildReference(parent, newLix, posp);
- }
-
- appliedRow = true;
- LOG.debug("Applied vectorizeLeftIndexingRow");
- }
- }
-
- if( isSingleRow && isSingleCol && !appliedRow )
- {
- //collect simple chains (w/o multiple consumers) of left indexing ops
- ArrayList<Hop> ihops = new ArrayList<Hop>();
- ihops.add(ihop0);
- Hop current = ihop0;
- while( current.getInput().get(0) instanceof LeftIndexingOp ) {
- LeftIndexingOp tmp = (LeftIndexingOp) current.getInput().get(0);
- if( tmp.getParent().size()>1 //multiple consumers, i.e., not a simple chain
- || !((LeftIndexingOp) tmp).getColLowerEqualsUpper() //row merge not applicable
- || tmp.getInput().get(4) != ihop0.getInput().get(4) //not the same col
- || tmp.getInput().get(0).getDim1() <= 1 ) //target is single row or unknown
- {
- break;
- }
- ihops.add( tmp );
- current = tmp;
- }
- //apply rewrite if found candidates
- if( ihops.size() > 1 ){
- Hop input = current.getInput().get(0);
- Hop colExpr = ihop0.getInput().get(4); //keep before reset
-
- //new row indexing operator
- IndexingOp newRix = new IndexingOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, input,
- new LiteralOp(1), HopRewriteUtils.createValueHop(input, true),
- colExpr, colExpr, false, true);
- HopRewriteUtils.setOutputParameters(newRix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newRix.refreshSizeInformation();
-
- //rewrite bottom left indexing operator
- HopRewriteUtils.removeChildReference(current, input); //input data
- HopRewriteUtils.addChildReference(current, newRix, 0);
-
- //reset row index all candidates
- for( Hop c : ihops ) {
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(4), 4); //col lower expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 4);
- HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(5), 5); //col upper expr
- HopRewriteUtils.addChildReference(c, new LiteralOp(1), 5);
- c.refreshSizeInformation();
- }
-
- //new row left indexing operator (for all parents, only intermediates are guaranteed to have 1 parent)
- //(note: it's important to clone the parent list before creating newLix on top of ihop0)
- ArrayList<Hop> ihop0parents = (ArrayList<Hop>) ihop0.getParent().clone();
- ArrayList<Integer> ihop0parentsPos = new ArrayList<Integer>();
- for( Hop parent : ihop0parents ) {
- int posp = HopRewriteUtils.getChildReferencePos(parent, ihop0);
- HopRewriteUtils.removeChildReferenceByPos(parent, ihop0, posp); //input data
- ihop0parentsPos.add(posp);
- }
-
- LeftIndexingOp newLix = new LeftIndexingOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, input, ihop0,
- new LiteralOp(1), HopRewriteUtils.createValueHop(input, true),
- colExpr, colExpr, false, true);
- HopRewriteUtils.setOutputParameters(newLix, -1, -1, input.getRowsInBlock(), input.getColsInBlock(), -1);
- newLix.refreshSizeInformation();
-
- for( int i=0; i<ihop0parentsPos.size(); i++ ) {
- Hop parent = ihop0parents.get(i);
- int posp = ihop0parentsPos.get(i);
- HopRewriteUtils.addChildReference(parent, newLix, posp);
- }
-
- appliedRow = true;
- LOG.debug("Applied vectorizeLeftIndexingCol");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
deleted file mode 100644
index 5e0e54f..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: Insert checkpointing operations for caching purposes. Currently, we
- * follow a heuristic of checkpointing (1) all variables used read-only in loops,
- * and (2) intermediates used by multiple consumers.
- *
- * TODO (2) implement injection for multiple consumers (local and global).
- *
- */
-public class RewriteInjectSparkLoopCheckpointing extends StatementBlockRewriteRule
-{
- private boolean _checkCtx = false;
-
- public RewriteInjectSparkLoopCheckpointing(boolean checkParForContext)
- {
- _checkCtx = checkParForContext;
- }
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus status)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- if( !OptimizerUtils.isSparkExecutionMode() )
- {
- ret.add(sb); // nothing to do here
- return ret; //return original statement block
- }
-
- //1) We currently add checkpoint operations without information about the global program structure,
- //this assumes that redundant checkpointing is prevented at runtime level (instruction-level)
- //2) Also, we do not take size information into account right now. This means that all candidates
- //are checkpointed even if they are only used by CP operations.
-
- int blocksize = status.getBlocksize(); //block size set by reblock rewrite
-
- //apply rewrite for while, for, and parfor (the decision for parfor loop bodies is deferred until parfor
- //optimization because otherwise we would prevent remote parfor)
- if( (sb instanceof WhileStatementBlock || sb instanceof ForStatementBlock) //incl parfor
- && (_checkCtx ? !status.isInParforContext() : true) )
- {
- //step 1: determine checkpointing candidates
- ArrayList<String> candidates = new ArrayList<String>();
- VariableSet read = sb.variablesRead();
- VariableSet updated = sb.variablesUpdated();
-
- for( String rvar : read.getVariableNames() )
- if( !updated.containsVariable(rvar) && read.getVariable(rvar).getDataType()==DataType.MATRIX )
- candidates.add(rvar);
-
- //step 2: insert statement block with checkpointing operations
- if( !candidates.isEmpty() ) //existing candidates
- {
- StatementBlock sb0 = new StatementBlock();
- sb0.setDMLProg(sb.getDMLProg());
- sb0.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- ArrayList<Hop> hops = new ArrayList<Hop>();
- VariableSet livein = new VariableSet();
- VariableSet liveout = new VariableSet();
- for( String var : candidates )
- {
- DataIdentifier dat = read.getVariable(var);
- DataOp tread = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, DataOpTypes.TRANSIENTREAD,
- dat.getFilename(), dat.getDim1(), dat.getDim2(), dat.getNnz(), blocksize, blocksize);
- tread.setRequiresCheckpoint( true );
- DataOp twrite = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, tread, DataOpTypes.TRANSIENTWRITE, null);
- HopRewriteUtils.setOutputParameters(twrite, dat.getDim1(), dat.getDim2(), blocksize, blocksize, dat.getNnz());
- hops.add(twrite);
- livein.addVariable(var, read.getVariable(var));
- liveout.addVariable(var, read.getVariable(var));
- }
- sb0.set_hops(hops);
- sb0.setLiveIn(livein);
- sb0.setLiveOut(liveout);
- ret.add(sb0);
-
- //maintain rewrite status
- status.setInjectedCheckpoints();
- }
- }
-
- //add original statement block to end
- ret.add(sb);
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
deleted file mode 100644
index 71d913d..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-
-/**
- * Rule: BlockSizeAndReblock. For all statement blocks, determine
- * "optimal" block size, and place reblock Hops. For now, we just
- * use BlockSize 1K x 1K and do reblock after Persistent Reads and
- * before Persistent Writes.
- */
-public class RewriteInjectSparkPReadCheckpointing extends HopRewriteRule
-{
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( !OptimizerUtils.isSparkExecutionMode() )
- return roots;
-
- if( roots == null )
- return null;
-
- //top-level hops never modified
- for( Hop h : roots )
- rInjectCheckpointAfterPRead(h);
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- //not applicable to predicates (we do not allow persistent reads there)
- return root;
- }
-
- /**
- *
- * @param hop
- * @throws HopsException
- */
- private void rInjectCheckpointAfterPRead( Hop hop )
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- // The reblocking is performed after transform, and hence checkpoint only non-transformed reads.
- if( (hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.PERSISTENTREAD && !HopRewriteUtils.hasTransformParents(hop))
- || (hop.requiresReblock())
- )
- {
- //make given hop for checkpointing (w/ default storage level)
- //note: we do not recursively process childs here in order to prevent unnecessary checkpoints
- hop.setRequiresCheckpoint(true);
- }
- else
- {
- //process childs
- if( hop.getInput() != null ) {
- //process all childs (prevent concurrent modification by index access)
- for( int i=0; i<hop.getInput().size(); i++ )
- rInjectCheckpointAfterPRead( hop.getInput().get(i) );
- }
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteMatrixMultChainOptimization.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteMatrixMultChainOptimization.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteMatrixMultChainOptimization.java
deleted file mode 100644
index c656f87..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteMatrixMultChainOptimization.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.utils.Explain;
-
-/**
- * Rule: Determine the optimal order of execution for a chain of
- * matrix multiplications Solution: Classic Dynamic Programming
- * Approach Currently, the approach based only on matrix dimensions
- * Goal: To reduce the number of computations in the run-time
- * (map-reduce) layer
- */
-public class RewriteMatrixMultChainOptimization extends HopRewriteRule
-{
-
- private static final Log LOG = LogFactory.getLog(RewriteMatrixMultChainOptimization.class.getName());
- private static final boolean LDEBUG = false;
-
- static
- {
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.rewrite.RewriteMatrixMultChainOptimization")
- .setLevel((Level) Level.TRACE);
- }
- }
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- for( Hop h : roots )
- {
- // Find the optimal order for the chain whose result is the current HOP
- rule_OptimizeMMChains(h);
- }
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return null;
-
- // Find the optimal order for the chain whose result is the current HOP
- rule_OptimizeMMChains(root);
-
- return root;
- }
-
- /**
- * rule_OptimizeMMChains(): This method recurses through all Hops in the DAG
- * to find chains that need to be optimized.
- */
- private void rule_OptimizeMMChains(Hop hop)
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- if ( hop instanceof AggBinaryOp && ((AggBinaryOp) hop).isMatrixMultiply()
- && !((AggBinaryOp)hop).hasLeftPMInput()
- && hop.getVisited() != Hop.VisitStatus.DONE )
- {
- // Try to find and optimize the chain in which current Hop is the
- // last operator
- optimizeMMChain(hop);
- }
-
- for (Hop hi : hop.getInput())
- rule_OptimizeMMChains(hi);
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
-
- /**
- * optimizeMMChain(): It optimizes the matrix multiplication chain in which
- * the last Hop is "this". Step-1) Identify the chain (mmChain). (Step-2) clear all
- * links among the Hops that are involved in mmChain. (Step-3) Find the
- * optimal ordering (dynamic programming) (Step-4) Relink the hops in
- * mmChain.
- */
- private void optimizeMMChain( Hop hop ) throws HopsException
- {
- if( LOG.isTraceEnabled() ) {
- LOG.trace("MM Chain Optimization for HOP: (" + " " + hop.getClass().getSimpleName() + ", " + hop.getHopID() + ", "
- + hop.getName() + ")");
- }
-
- ArrayList<Hop> mmChain = new ArrayList<Hop>();
- ArrayList<Hop> mmOperators = new ArrayList<Hop>();
- ArrayList<Hop> tempList;
-
- // Step 1: Identify the chain (mmChain) & clear all links among the Hops
- // that are involved in mmChain.
-
- mmOperators.add(hop);
- // Initialize mmChain with my inputs
- for (Hop hi : hop.getInput()) {
- mmChain.add(hi);
- }
-
- // expand each Hop in mmChain to find the entire matrix multiplication
- // chain
- int i = 0;
- while (i < mmChain.size()) {
-
- boolean expandable = false;
-
- Hop h = mmChain.get(i);
- /*
- * Check if mmChain[i] is expandable:
- * 1) It must be MATMULT
- * 2) It must not have been visited already
- * (one MATMULT should get expanded only in one chain)
- * 3) Its output should not be used in multiple places
- * (either within chain or outside the chain)
- */
-
- if ( h instanceof AggBinaryOp && ((AggBinaryOp) h).isMatrixMultiply()
- && !((AggBinaryOp)hop).hasLeftPMInput()
- && h.getVisited() != Hop.VisitStatus.DONE )
- {
- // check if the output of "h" is used at multiple places. If yes, it can
- // not be expanded.
- if (h.getParent().size() > 1 || inputCount( (Hop) ((h.getParent().toArray())[0]), h) > 1 ) {
- expandable = false;
- break;
- }
- else
- expandable = true;
- }
-
- h.setVisited(Hop.VisitStatus.DONE);
-
- if ( !expandable ) {
- i = i + 1;
- } else {
- tempList = mmChain.get(i).getInput();
- if (tempList.size() != 2) {
- throw new HopsException(hop.printErrorLocation() + "Hops::rule_OptimizeMMChain(): AggBinary must have exactly two inputs.");
- }
-
- // add current operator to mmOperators, and its input nodes to mmChain
- mmOperators.add(mmChain.get(i));
- mmChain.set(i, tempList.get(0));
- mmChain.add(i + 1, tempList.get(1));
- }
- }
-
- // print the MMChain
- if( LOG.isTraceEnabled() ) {
- LOG.trace("Identified MM Chain: ");
- for (Hop h : mmChain) {
- logTraceHop(h, 1);
- }
- }
-
- if (mmChain.size() == 2) {
- // If the chain size is 2, then there is nothing to optimize.
- return;
- }
- else
- {
- // Step 2: construct dims array
- double[] dimsArray = new double[mmChain.size() + 1];
- boolean dimsKnown = getDimsArray( hop, mmChain, dimsArray );
-
- if( dimsKnown ) {
- // Step 3: clear the links among Hops within the identified chain
- clearLinksWithinChain ( hop, mmOperators );
-
- // Step 4: Find the optimal ordering via dynamic programming.
-
- // Invoke Dynamic Programming
- int size = mmChain.size();
- int[][] split = mmChainDP(dimsArray, mmChain.size());
-
- // Step 5: Relink the hops using the optimal ordering (split[][]) found from DP.
- LOG.trace("Optimal MM Chain: ");
- mmChainRelinkHops(mmOperators.get(0), 0, size - 1, mmChain, mmOperators, 1, split, 1);
- }
- }
- }
-
- /**
- * mmChainDP(): Core method to perform dynamic programming on a given array
- * of matrix dimensions.
- *
- * Thomas H. Cormen, Charles E. Leiserson, Ronald L. Rivest, Clifford Stein
- * Introduction to Algorithms, Third Edition, MIT Press, page 395.
- */
- private int[][] mmChainDP(double[] dimArray, int size)
- {
- double[][] dpMatrix = new double[size][size]; //min cost table
- int[][] split = new int[size][size]; //min cost index table
-
- //init minimum costs for chains of length 1
- for (int i = 0; i < size; i++) {
- Arrays.fill(dpMatrix[i], 0);
- Arrays.fill(split[i], -1);
- }
-
- //compute cost-optimal chains for increasing chain sizes
- for (int l = 2; l <= size; l++) { // chain length
- for (int i = 0; i < size - l + 1; i++) {
- int j = i + l - 1;
- // find cost of (i,j)
- dpMatrix[i][j] = Double.MAX_VALUE;
- for (int k = i; k <= j - 1; k++)
- {
- //recursive cost computation
- double cost = dpMatrix[i][k] + dpMatrix[k + 1][j]
- + (dimArray[i] * dimArray[k + 1] * dimArray[j + 1]);
-
- //prune suboptimal
- if (cost < dpMatrix[i][j]) {
- dpMatrix[i][j] = cost;
- split[i][j] = k;
- }
- }
-
- if( LOG.isTraceEnabled() ){
- LOG.trace("mmchainopt [i="+(i+1)+",j="+(j+1)+"]: costs = "+dpMatrix[i][j]+", split = "+(split[i][j]+1));
- }
- }
- }
-
- return split;
- }
-
- /**
- * mmChainRelinkHops(): This method gets invoked after finding the optimal
- * order (split[][]) from dynamic programming. It relinks the Hops that are
- * part of the mmChain. mmChain : basic operands in the entire matrix
- * multiplication chain. mmOperators : Hops that store the intermediate
- * results in the chain. For example: A = B %*% (C %*% D) there will be
- * three Hops in mmChain (B,C,D), and two Hops in mmOperators (one for each
- * %*%) .
- */
- private void mmChainRelinkHops(Hop h, int i, int j, ArrayList<Hop> mmChain, ArrayList<Hop> mmOperators,
- int opIndex, int[][] split, int level)
- {
- //single matrix - end of recursion
- if (i == j) {
- logTraceHop(h, level);
- return;
- }
-
- if( LOG.isTraceEnabled() ){
- String offset = Explain.getIdentation(level);
- LOG.trace(offset + "(");
- }
-
- // Set Input1 for current Hop h
- if (i == split[i][j]) {
- h.getInput().add(mmChain.get(i));
- mmChain.get(i).getParent().add(h);
- } else {
- h.getInput().add(mmOperators.get(opIndex));
- mmOperators.get(opIndex).getParent().add(h);
- opIndex = opIndex + 1;
- }
-
- // Set Input2 for current Hop h
- if (split[i][j] + 1 == j) {
- h.getInput().add(mmChain.get(j));
- mmChain.get(j).getParent().add(h);
- } else {
- h.getInput().add(mmOperators.get(opIndex));
- mmOperators.get(opIndex).getParent().add(h);
- opIndex = opIndex + 1;
- }
-
- // Find children for both the inputs
- mmChainRelinkHops(h.getInput().get(0), i, split[i][j], mmChain, mmOperators, opIndex, split, level+1);
- mmChainRelinkHops(h.getInput().get(1), split[i][j] + 1, j, mmChain, mmOperators, opIndex, split, level+1);
-
- // Propagate properties of input hops to current hop h
- h.refreshSizeInformation();
-
- if( LOG.isTraceEnabled() ){
- String offset = Explain.getIdentation(level);
- LOG.trace(offset + ")");
- }
- }
-
- /**
- *
- * @param operators
- * @throws HopsException
- */
- private void clearLinksWithinChain ( Hop hop, ArrayList<Hop> operators )
- throws HopsException
- {
- Hop op, input1, input2;
-
- for ( int i=0; i < operators.size(); i++ ) {
- op = operators.get(i);
- if ( op.getInput().size() != 2 || (i != 0 && op.getParent().size() > 1 ) ) {
- throw new HopsException(hop.printErrorLocation() + "Unexpected error while applying optimization on matrix-mult chain. \n");
- }
- input1 = op.getInput().get(0);
- input2 = op.getInput().get(1);
-
- op.getInput().clear();
- input1.getParent().remove(op);
- input2.getParent().remove(op);
- }
- }
-
- /**
- * Obtains all dimension information of the chain and constructs the dimArray.
- * If all dimensions are known it returns true; othrewise the mmchain rewrite
- * should be ended without modifications.
- *
- * @param hop
- * @param chain
- * @param dimArray
- * @return
- * @throws HopsException
- */
- private boolean getDimsArray( Hop hop, ArrayList<Hop> chain, double[] dimsArray )
- throws HopsException
- {
- boolean dimsKnown = true;
-
- // Build the array containing dimensions from all matrices in the chain
- // check the dimensions in the matrix chain to insure all dimensions are known
- for (int i=0; i< chain.size(); i++){
- if (chain.get(i).getDim1() <= 0 || chain.get(i).getDim2() <= 0)
- dimsKnown = false;
- }
-
- if( dimsKnown ) { //populate dims array if all dims known
- for (int i = 0; i < chain.size(); i++)
- {
- if (i == 0) {
- dimsArray[i] = chain.get(i).getDim1();
- if (dimsArray[i] <= 0) {
- throw new HopsException(hop.printErrorLocation() +
- "Hops::optimizeMMChain() : Invalid Matrix Dimension: "+ dimsArray[i]);
- }
- } else {
- if (chain.get(i - 1).getDim2() != chain.get(i).getDim1()) {
- throw new HopsException(hop.printErrorLocation() +
- "Hops::optimizeMMChain() : Matrix Dimension Mismatch: "+chain.get(i - 1).getDim2()+" != "+chain.get(i).getDim1());
- }
- }
- dimsArray[i + 1] = chain.get(i).getDim2();
- if (dimsArray[i + 1] <= 0) {
- throw new HopsException(hop.printErrorLocation() +
- "Hops::optimizeMMChain() : Invalid Matrix Dimension: " + dimsArray[i + 1]);
- }
- }
- }
-
- return dimsKnown;
- }
-
-
- /**
- *
- * @param p
- * @param h
- * @return
- */
- private int inputCount ( Hop p, Hop h ) {
- int count = 0;
- for ( int i=0; i < p.getInput().size(); i++ )
- if ( p.getInput().get(i).equals(h) )
- count++;
- return count;
- }
-
- /**
- *
- * @param hop
- * @param level
- */
- private void logTraceHop( Hop hop, int level )
- {
- if( LOG.isTraceEnabled() ) {
- String offset = Explain.getIdentation(level);
- LOG.trace(offset+ "Hop " + hop.getName() + "(" + hop.getClass().getSimpleName() + ", " + hop.getHopID() + ")" + " "
- + hop.getDim1() + "x" + hop.getDim2());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemovePersistentReadWrite.java
deleted file mode 100644
index 4eb67b6..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-
-/**
- * This rewrite is a custom rewrite for JMLC in order to replace all persistent reads
- * and writes with transient reads and writes from the symbol table.
- *
- *
- */
-public class RewriteRemovePersistentReadWrite extends HopRewriteRule
-{
-
- private HashSet<String> _inputs = null;
- private HashSet<String> _outputs = null;
-
- public RewriteRemovePersistentReadWrite( String[] in, String[] out )
- {
- _inputs = new HashSet<String>();
- for( String var : in )
- _inputs.add( var );
- _outputs = new HashSet<String>();
- for( String var : out )
- _outputs.add( var );
- }
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- for( Hop h : roots )
- rule_RemovePersistentDataOp( h );
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return root;
-
- rule_RemovePersistentDataOp( root );
-
- return root;
- }
-
- /**
- *
- * @param hop
- * @throws HopsException
- */
- private void rule_RemovePersistentDataOp( Hop hop )
- throws HopsException
- {
- //check mark processed
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //recursively process childs
- ArrayList<Hop> inputs = hop.getInput();
- for( int i=0; i<inputs.size(); i++ )
- rule_RemovePersistentDataOp( inputs.get(i) );
-
- //remove cast if unnecessary
- if( hop instanceof DataOp )
- {
- DataOp dop = (DataOp) hop;
- DataOpTypes dotype = dop.getDataOpType();
-
- switch( dotype )
- {
- case PERSISTENTREAD:
- if( _inputs.contains(dop.getName()) )
- dop.setDataOpType(DataOpTypes.TRANSIENTREAD);
- break;
- case PERSISTENTWRITE:
- if( _outputs.contains(dop.getName()) )
- dop.setDataOpType(DataOpTypes.TRANSIENTWRITE);
- break;
- default:
- //do nothing
- }
- }
-
- //mark processed
- hop.setVisited( VisitStatus.DONE );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveReadAfterWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveReadAfterWrite.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveReadAfterWrite.java
deleted file mode 100644
index a37c54a..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveReadAfterWrite.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.DataExpression;
-
-/**
- * Rule: RemoveReadAfterWrite. If there is a persistent read with the same filename
- * as a persistent write, and read has a higher line number than the write,
- * we remove the read and consume the write input directly. This is important for two
- * reasons (1) correctness and (2) performance. Without this rewrite, we could not
- * guarantee the order of read-after-write because there is not data dependency
- *
- */
-public class RewriteRemoveReadAfterWrite extends HopRewriteRule
-{
-
- @Override
- @SuppressWarnings("unchecked")
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- //collect all persistent reads and writes
- HashMap<String,Hop> reads = new HashMap<String,Hop>();
- HashMap<String,Hop> writes = new HashMap<String,Hop>();
- for( Hop h : roots )
- collectPersistentReadWriteOps( h, writes, reads );
-
- Hop.resetVisitStatus(roots);
-
- //check persistent reads for read-after-write pattern
- for( Entry<String, Hop> e : reads.entrySet() )
- {
- String rfname = e.getKey();
- Hop rhop = e.getValue();
- if( writes.containsKey(rfname) //same persistent filename
- && (writes.get(rfname).getBeginLine()<rhop.getBeginLine() //read after write
- || writes.get(rfname).getEndLine()<rhop.getEndLine()) ) //note: account for bug in line handling, TODO remove after line handling resolved
- {
- //rewire read consumers to write input
- Hop input = writes.get(rfname).getInput().get(0);
- ArrayList<Hop> parents = (ArrayList<Hop>) rhop.getParent().clone();
- for( Hop p : parents ) {
- int pos = HopRewriteUtils.getChildReferencePos(p, rhop);
- HopRewriteUtils.removeChildReferenceByPos(p, rhop, pos);
- HopRewriteUtils.addChildReference(p, input, pos);
- }
- }
- }
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- //do noting, read/write do not occur in predicates
-
- return root;
- }
-
- /**
- *
- * @param hop
- * @param pWrites
- * @param pReads
- * @throws HopsException
- */
- private void collectPersistentReadWriteOps(Hop hop, HashMap<String,Hop> pWrites, HashMap<String,Hop> pReads)
- throws HopsException
- {
- if( hop.getVisited() == Hop.VisitStatus.DONE )
- return;
-
- //process childs
- if( !hop.getInput().isEmpty() )
- for( Hop c : hop.getInput() )
- collectPersistentReadWriteOps(c, pWrites, pReads);
-
- //process current hop
- if( hop instanceof DataOp )
- {
- DataOp dop = (DataOp)hop;
- if( dop.getDataOpType()==DataOpTypes.PERSISTENTREAD )
- pReads.put(dop.getFileName(), dop);
- else if( dop.getDataOpType()==DataOpTypes.PERSISTENTWRITE )
- {
- Hop fname = dop.getInput().get(dop.getParameterIndex(DataExpression.IO_FILENAME));
- if( fname instanceof LiteralOp ) //only constant writes
- pWrites.put(((LiteralOp) fname).getStringValue(), dop);
- }
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
deleted file mode 100644
index 0587729..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-
-/**
- * Rule: Simplify program structure by pulling if or else statement body out
- * (removing the if statement block ifself) in order to allow intra-procedure
- * analysis to propagate exact statistics.
- *
- */
-public class RewriteRemoveUnnecessaryBranches extends StatementBlockRewriteRule
-{
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- if( sb instanceof IfStatementBlock )
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- Hop pred = isb.getPredicateHops();
-
- //apply rewrite if literal op (constant value)
- if( pred instanceof LiteralOp )
- {
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- LiteralOp litpred = (LiteralOp) pred;
- boolean condition = HopRewriteUtils.getBooleanValue(litpred);
-
- if( condition )
- {
- //pull-out simple if body
- if( !istmt.getIfBody().isEmpty() )
- ret.addAll( istmt.getIfBody() ); //pull if-branch
- //otherwise: add nothing (remove if-else)
- }
- else
- {
- //pull-out simple else body
- if( !istmt.getElseBody().isEmpty() )
- ret.addAll( istmt.getElseBody() ); //pull else-branch
- //otherwise: add nothing (remove if-else)
- }
-
- state.setRemovedBranches();
- LOG.debug("Applied removeUnnecessaryBranches (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+").");
- }
- else //keep original sb (non-constant condition)
- ret.add( sb );
- }
- else //keep original sb (no if)
- ret.add( sb );
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
deleted file mode 100644
index 011f67e..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: RemoveUnnecessaryCasts. For all value type casts check
- * if they are really necessary. If both cast input and output
- * type are the same, the cast itself is redundant.
- *
- * There are two use case where this can arise: (1) automatically
- * inserted casts on function inlining (in case of unknown value
- * types), and (2) explicit script-level value type casts, that
- * might be redundant according to the read input data.
- *
- * The benefit of this rewrite is negligible for scalars. However,
- * when we support matrices with different value types, those casts
- * might refer to matrices and with that incur large costs.
- *
- */
-public class RewriteRemoveUnnecessaryCasts extends HopRewriteRule
-{
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- for( Hop h : roots )
- rule_RemoveUnnecessaryCasts( h );
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return root;
-
- rule_RemoveUnnecessaryCasts( root );
-
- return root;
- }
-
- /**
- *
- * @param hop
- */
- private void rule_RemoveUnnecessaryCasts( Hop hop )
- {
- //check mark processed
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //recursively process childs
- ArrayList<Hop> inputs = hop.getInput();
- for( int i=0; i<inputs.size(); i++ )
- rule_RemoveUnnecessaryCasts( inputs.get(i) );
-
- //remove cast if unnecessary
- if( hop instanceof UnaryOp && HopRewriteUtils.isValueTypeCast(((UnaryOp)hop).getOp()) )
- {
- Hop in = hop.getInput().get(0);
- ValueType vtIn = in.getValueType(); //type cast input
- ValueType vtOut = hop.getValueType(); //type cast output
-
- //if input/output types match, no need to cast
- if( vtIn == vtOut && vtIn != ValueType.UNKNOWN )
- {
- ArrayList<Hop> parents = hop.getParent();
- for( int i=0; i<parents.size(); i++ ) //for all parents
- {
- Hop p = parents.get(i);
- ArrayList<Hop> pin = p.getInput();
- for( int j=0; j<pin.size(); j++ ) //for all parent childs
- {
- Hop pinj = pin.get(j);
- if( pinj == hop ) //found parent ref
- {
- //rehang cast input as child of cast consumer
- pin.remove( j ); //remove cast ref
- pin.add(j, in); //add ref to cast input
- in.getParent().remove(hop); //remove cast from cast input parents
- in.getParent().add( p ); //add parent to cast input parents
- }
- }
- }
- parents.clear();
- }
- }
-
- //mark processed
- hop.setVisited( VisitStatus.DONE );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
deleted file mode 100644
index 2920853..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
+++ /dev/null
@@ -1,519 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.OpOp3;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-
-/**
- * Rule: Split Hop DAG after specific data-dependent operators. This is
- * important to create recompile hooks if output dimensions are usually
- * significantly overestimated.
- *
- * This is a recursive statementblock rewrite rule.
- *
- * NOTE: Before we used AssignmentStatement.controlStatement() in order to force
- * statementblock cuts. However, this (1) cuts not only after but before-and-after
- * (which prevents certain rewrites because the input operators are unknown),
- * and (2) is statement-centric which potentially prevents the cut right after
- * the problematic operation.
- *
- * TODO: Cleanup runtime to never access individual statements of potentially
- * split statements blocks again (for consistency). However, currently it is
- * only used in places (e.g., parfor optimizer) that are not directly affected.
- */
-public class RewriteSplitDagDataDependentOperators extends StatementBlockRewriteRule
-{
-
- private static String _varnamePredix = "_sbcvar";
- private static IDSequence _seq = new IDSequence();
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- //collect all unknown csv reads hops
- ArrayList<Hop> cand = new ArrayList<Hop>();
- collectDataDependentOperators( sb.get_hops(), cand );
- Hop.resetVisitStatus(sb.get_hops());
-
- //split hop dag on demand
- if( !cand.isEmpty() )
- {
- //collect child operators of candidates (to prevent rewrite anomalies)
- HashSet<Hop> candChilds = new HashSet<Hop>();
- collectCandidateChildOperators( cand, candChilds );
-
- try
- {
- //duplicate sb incl live variable sets
- StatementBlock sb1 = new StatementBlock();
- sb1.setDMLProg(sb.getDMLProg());
- sb1.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- sb1.setLiveIn(new VariableSet());
- sb1.setLiveOut(new VariableSet());
-
- //move data-dependent ops incl transient writes to new statement block
- //(and replace original persistent read with transient read)
- ArrayList<Hop> sb1hops = new ArrayList<Hop>();
- for( Hop c : cand )
- {
- //if there are already transient writes use them and don't introduce artificial variables;
- //unless there are transient reads w/ the same variable name in the current dag which can
- //lead to invalid reordering if variable consumers are not feeding into the candidate op.
- boolean hasTWrites = hasTransientWriteParents(c);
- boolean moveTWrite = hasTWrites ? HopRewriteUtils.rHasSimpleReadChain(c,
- getFirstTransientWriteParent(c).getName()) : false;
-
- String varname = null;
- long rlen = c.getDim1();
- long clen = c.getDim2();
- long nnz = c.getNnz();
- long brlen = c.getRowsInBlock();
- long bclen = c.getColsInBlock();
-
- if( hasTWrites && moveTWrite) //reuse existing transient_write
- {
- Hop twrite = getFirstTransientWriteParent(c);
- varname = twrite.getName();
-
- //create new transient read
- DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(),
- DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, brlen, bclen);
- tread.setVisited(VisitStatus.DONE);
- HopRewriteUtils.copyLineNumbers(c, tread);
-
- //replace data-dependent operator with transient read
- ArrayList<Hop> parents = new ArrayList<Hop>(c.getParent());
- for( int i=0; i<parents.size(); i++ ) {
- //prevent concurrent modification by index access
- Hop parent = parents.get(i);
- if( !candChilds.contains(parent) ) //anomaly filter
- {
- if( parent != twrite ) {
- int pos = HopRewriteUtils.getChildReferencePos(parent, c);
- HopRewriteUtils.removeChildReferenceByPos(parent, c, pos);
- HopRewriteUtils.addChildReference(parent, tread, pos);
- }
- else
- sb.get_hops().remove(parent);
- }
- }
-
- //add data-dependent operator sub dag to first statement block
- sb1hops.add(twrite);
- }
- else //create transient write to artificial variables
- {
- varname = _varnamePredix + _seq.getNextID();
-
- //create new transient read
- DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(),
- DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, brlen, bclen);
- tread.setVisited(VisitStatus.DONE);
- HopRewriteUtils.copyLineNumbers(c, tread);
-
- //replace data-dependent operator with transient read
- ArrayList<Hop> parents = new ArrayList<Hop>(c.getParent());
- for( int i=0; i<parents.size(); i++ ) {
- //prevent concurrent modification by index access
- Hop parent = parents.get(i);
- if( !candChilds.contains(parent) ) //anomaly filter
- {
- int pos = HopRewriteUtils.getChildReferencePos(parent, c);
- HopRewriteUtils.removeChildReferenceByPos(parent, c, pos);
- HopRewriteUtils.addChildReference(parent, tread, pos);
- }
- }
-
- //add data-dependent operator sub dag to first statement block
- DataOp twrite = new DataOp(varname, c.getDataType(), c.getValueType(),
- c, DataOpTypes.TRANSIENTWRITE, null);
- twrite.setVisited(VisitStatus.DONE);
- twrite.setOutputParams(rlen, clen, nnz, brlen, bclen);
- HopRewriteUtils.copyLineNumbers(c, twrite);
- sb1hops.add(twrite);
- }
-
- //update live in and out of new statement block (for piggybacking)
- DataIdentifier diVar = new DataIdentifier(varname);
- diVar.setDimensions(rlen, clen);
- diVar.setBlockDimensions(brlen, bclen);
- diVar.setDataType(c.getDataType());
- diVar.setValueType(c.getValueType());
- sb1.liveOut().addVariable(varname, new DataIdentifier(diVar));
- sb.liveIn().addVariable(varname, new DataIdentifier(diVar));
- }
-
- //ensure disjoint operators across DAGs (prevent replicated operations)
- handleReplicatedOperators( sb1hops, sb.get_hops(), sb1.liveOut(), sb.liveIn() );
-
- //deep copy new dag (in order to prevent any dangling references)
- sb1.set_hops(Recompiler.deepCopyHopsDag(sb1hops));
- sb1.updateRecompilationFlag();
-
- //recursive application of rewrite rule (in case of multiple data dependent operators
- //with data dependencies in between each other)
- ArrayList<StatementBlock> tmp = rewriteStatementBlock( sb1, state);
-
- //add new statement blocks to output
- ret.addAll(tmp); //statement block with data dependent hops
- ret.add(sb); //statement block with remaining hops
- }
- catch(Exception ex)
- {
- throw new HopsException("Failed to split hops dag for data dependent operators with unknown size.", ex);
- }
-
- LOG.debug("Applied splitDagDataDependentOperators (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+").");
- }
- //keep original hop dag
- else
- {
- ret.add(sb);
- }
-
- return ret;
- }
-
- /**
- *
- * @param roots
- * @param cand
- */
- private void collectDataDependentOperators( ArrayList<Hop> roots, ArrayList<Hop> cand )
- {
- if( roots == null )
- return;
-
- Hop.resetVisitStatus(roots);
- for( Hop root : roots )
- rCollectDataDependentOperators(root, cand);
- }
-
- /**
- *
- * @param root
- * @param cand
- */
- private void rCollectDataDependentOperators( Hop hop, ArrayList<Hop> cand )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //prevent unnecessary dag split (dims known or no consumer operations)
- boolean noSplitRequired = ( hop.dimsKnown() || HopRewriteUtils.hasOnlyWriteParents(hop, true, true) );
- boolean investigateChilds = true;
-
- //collect data dependent operations (to be extended as necessary)
- //#1 removeEmpty
- if( hop instanceof ParameterizedBuiltinOp
- && ((ParameterizedBuiltinOp) hop).getOp()==ParamBuiltinOp.RMEMPTY
- && !noSplitRequired
- && !(hop.getParent().size()==1 && hop.getParent().get(0) instanceof TernaryOp
- && ((TernaryOp)hop.getParent().get(0)).isMatrixIgnoreZeroRewriteApplicable()))
- {
- ParameterizedBuiltinOp pbhop = (ParameterizedBuiltinOp)hop;
- cand.add(pbhop);
- investigateChilds = false;
-
- //keep interesting consumer information, flag hops accordingly
- boolean noEmptyBlocks = true;
- boolean onlyPMM = true;
- boolean diagInput = pbhop.isTargetDiagInput();
- for( Hop p : hop.getParent() ) {
- //list of operators without need for empty blocks to be extended as needed
- noEmptyBlocks &= ( p instanceof AggBinaryOp && hop == p.getInput().get(0)
- || p instanceof UnaryOp && ((UnaryOp)p).getOp()==OpOp1.NROW);
- onlyPMM &= (p instanceof AggBinaryOp && hop == p.getInput().get(0));
- }
- pbhop.setOutputEmptyBlocks(!noEmptyBlocks);
-
- if( onlyPMM && diagInput ){
- //configure rmEmpty to directly output selection vector
- //(only applied if dynamic recompilation enabled)
-
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
- pbhop.setOutputPermutationMatrix(true);
- for( Hop p : hop.getParent() )
- ((AggBinaryOp)p).setHasLeftPMInput(true);
- }
- }
-
- //#2 ctable with unknown dims
- if( hop instanceof TernaryOp
- && ((TernaryOp) hop).getOp()==OpOp3.CTABLE
- && hop.getInput().size() < 4 //dims not provided
- && !noSplitRequired )
- {
- cand.add(hop);
- investigateChilds = false;
-
- //keep interesting consumer information, flag hops accordingly
- boolean onlyPMM = true;
- for( Hop p : hop.getParent() ) {
- onlyPMM &= (p instanceof AggBinaryOp && hop == p.getInput().get(0));
- }
-
- if( onlyPMM && HopRewriteUtils.isBasic1NSequence(hop.getInput().get(0)) )
- hop.setOutputEmptyBlocks(false);
- }
-
- //#3 orderby childs computed in same DAG
- if( hop instanceof ReorgOp
- && ((ReorgOp)hop).getOp()==ReOrgOp.SORT )
- {
- //params 'decreasing' / 'indexreturn'
- for( int i=2; i<=3; i++ ) {
- Hop c = hop.getInput().get(i);
- if( !(c instanceof LiteralOp || c instanceof DataOp) ){
- cand.add(c);
- c.setVisited(VisitStatus.DONE);
- investigateChilds = false;
- }
-
- }
- }
-
- //process children (if not already found a special operators;
- //otherwise, processed by recursive rule application)
- if( investigateChilds )
- if( hop.getInput()!=null )
- for( Hop c : hop.getInput() )
- rCollectDataDependentOperators(c, cand);
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- *
- * @param hop
- * @return
- */
- private boolean hasTransientWriteParents( Hop hop )
- {
- for( Hop p : hop.getParent() )
- if( p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.TRANSIENTWRITE )
- return true;
- return false;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- private Hop getFirstTransientWriteParent( Hop hop )
- {
- for( Hop p : hop.getParent() )
- if( p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.TRANSIENTWRITE )
- return p;
- return null;
- }
-
- /**
- *
- * @param rootsSB1
- * @param rootsSB2
- * @param candChilds
- * @param cand
- * @param sb2in
- * @param sb1out
- */
- private void handleReplicatedOperators( ArrayList<Hop> rootsSB1, ArrayList<Hop> rootsSB2, VariableSet sb1out, VariableSet sb2in )
- {
- //step 1: create probe set SB1
- HashSet<Hop> probeSet = new HashSet<Hop>();
- Hop.resetVisitStatus(rootsSB1);
- for( Hop h : rootsSB1 )
- rAddHopsToProbeSet( h, probeSet );
-
- //step 2: probe SB2 operators top-down (collect cut candidates)
- HashSet<Pair<Hop,Hop>> candSet = new HashSet<Pair<Hop,Hop>>();
- Hop.resetVisitStatus(rootsSB2);
- for( Hop h : rootsSB2 )
- rProbeAndAddHopsToCandidateSet(h, probeSet, candSet);
-
- //step 3: create additional cuts
- for( Pair<Hop,Hop> p : candSet )
- {
- String varname = _varnamePredix + _seq.getNextID();
-
- Hop hop = p.getKey();
- Hop c = p.getValue();
-
- DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, c.getDim1(), c.getDim2(), c.getNnz(), c.getRowsInBlock(), c.getColsInBlock());
- tread.setVisited(VisitStatus.DONE);
- HopRewriteUtils.copyLineNumbers(c, tread);
-
- DataOp twrite = new DataOp(varname, c.getDataType(), c.getValueType(), c, DataOpTypes.TRANSIENTWRITE, null);
- twrite.setVisited(VisitStatus.DONE);
- twrite.setOutputParams(c.getDim1(), c.getDim2(), c.getNnz(), c.getRowsInBlock(), c.getColsInBlock());
- HopRewriteUtils.copyLineNumbers(c, twrite);
-
- //create additional cut by rewriting both hop dags
- int pos = HopRewriteUtils.getChildReferencePos(hop, c);
- HopRewriteUtils.removeChildReferenceByPos(hop, c, pos);
- HopRewriteUtils.addChildReference(hop, tread, pos);
-
- //update live in and out of new statement block (for piggybacking)
- DataIdentifier diVar = new DataIdentifier(varname);
- diVar.setDimensions(c.getDim1(), c.getDim2());
- diVar.setBlockDimensions(c.getRowsInBlock(), c.getColsInBlock());
- diVar.setDataType(c.getDataType());
- diVar.setValueType(c.getValueType());
- sb1out.addVariable(varname, new DataIdentifier(diVar));
- sb2in.addVariable(varname, new DataIdentifier(diVar));
-
- rootsSB1.add(twrite);
- }
- }
-
- /**
- *
- * @param hop
- * @param probeSet
- * @param candChilds
- * @param cand
- */
- private void rAddHopsToProbeSet( Hop hop, HashSet<Hop> probeSet )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //prevent cuts for no-ops
- if( !( (hop instanceof DataOp && !((DataOp)hop).isPersistentReadWrite() )
- || hop instanceof LiteralOp) )
- {
- probeSet.add(hop);
- }
-
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- rAddHopsToProbeSet(c, probeSet);
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- *
- *
- * NOTE: candset is a set of parent-child pairs because a parent might have
- * multiple references to replicated hops.
- *
- * @param hop
- * @param probeSet
- * @param candSet
- */
- private void rProbeAndAddHopsToCandidateSet( Hop hop, HashSet<Hop> probeSet, HashSet<Pair<Hop,Hop>> candSet )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() ) {
- //probe for replicated operator, if any child is replicated, keep parent
- //for cut between parent-child; otherwise recursively descend.
- if( !probeSet.contains(c) )
- rProbeAndAddHopsToCandidateSet(c, probeSet, candSet);
- else
- {
- candSet.add(new Pair<Hop,Hop>(hop,c));
- }
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- *
- * @param cand
- * @param candChilds
- */
- private void collectCandidateChildOperators( ArrayList<Hop> cand, HashSet<Hop> candChilds )
- {
- Hop.resetVisitStatus(cand);
- if( cand != null )
- for( Hop root : cand )
- rCollectCandidateChildOperators(root, cand, candChilds, false);
-
- // Immediately reset the visit status because candidates might be inner nodes in the DAG.
- // Subsequent resets on the root nodes of the DAG would otherwise not necessarily reach
- // these nodes which could lead to missing checks on subsequent passes (e.g., when checking
- // for replicated operators).
- Hop.resetVisitStatus(cand);
- }
-
- /**
- *
- * @param hop
- * @param cand
- * @param candChilds
- * @param collect
- */
- private void rCollectCandidateChildOperators( Hop hop, ArrayList<Hop> cand, HashSet<Hop> candChilds, boolean collect )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //collect operator if necessary
- if( collect ) {
- candChilds.add(hop);
- }
-
- //activate collection if we passed a candidate
- boolean passedFlag = collect;
- if( cand.contains(hop) ) {
- passedFlag = true;
- }
-
- //process childs recursively
- if( hop.getInput()!=null ) {
- for( Hop c : hop.getInput() )
- rCollectCandidateChildOperators(c, cand, candChilds, passedFlag);
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-}
[15/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/StatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/StatementBlock.java
deleted file mode 100644
index 437d859..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/StatementBlock.java
+++ /dev/null
@@ -1,1063 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.FormatType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-
-
-public class StatementBlock extends LiveVariableAnalysis
-{
-
- protected static final Log LOG = LogFactory.getLog(StatementBlock.class.getName());
- protected static IDSequence _seq = new IDSequence();
-
- protected DMLProgram _dmlProg;
- protected ArrayList<Statement> _statements;
- ArrayList<Hop> _hops = null;
- ArrayList<Lop> _lops = null;
- HashMap<String,ConstIdentifier> _constVarsIn;
- HashMap<String,ConstIdentifier> _constVarsOut;
-
- private boolean _requiresRecompile = false;
-
- public StatementBlock(){
- _dmlProg = null;
- _statements = new ArrayList<Statement>();
- _read = new VariableSet();
- _updated = new VariableSet();
- _gen = new VariableSet();
- _kill = new VariableSet();
- _warnSet = new VariableSet();
- _initialized = true;
- _constVarsIn = new HashMap<String,ConstIdentifier>();
- _constVarsOut = new HashMap<String,ConstIdentifier>();
- }
-
- public void setDMLProg(DMLProgram dmlProg){
- _dmlProg = dmlProg;
- }
-
- public DMLProgram getDMLProg(){
- return _dmlProg;
- }
-
- public void addStatement(Statement s){
- _statements.add(s);
-
- if (_statements.size() == 1){
- this._filename = s.getFilename();
- this._beginLine = s.getBeginLine();
- this._beginColumn = s.getBeginColumn();
- }
-
- this._endLine = s.getEndLine();
- this._endColumn = s.getEndColumn();
-
- }
-
- /**
- * replace statement
- */
- public void replaceStatement(int index, Statement passedStmt){
- this._statements.set(index, passedStmt);
-
- if (index == 0){
- this._beginLine = passedStmt.getBeginLine();
- this._beginColumn = passedStmt.getBeginColumn();
- }
-
- else if (index == this._statements.size() -1){
- this._endLine = passedStmt.getEndLine();
- this._endColumn = passedStmt.getEndColumn();
- }
- }
-
- public void addStatementBlock(StatementBlock s){
- for (int i = 0; i < s.getNumStatements(); i++){
- _statements.add(s.getStatement(i));
- }
-
- this._beginLine = _statements.get(0).getBeginLine();
- this._beginColumn = _statements.get(0).getBeginColumn();
-
- this._endLine = _statements.get(_statements.size() - 1).getEndLine();
- this._endColumn = _statements.get(_statements.size() - 1).getEndColumn();
- }
-
- public int getNumStatements(){
- return _statements.size();
- }
-
- public Statement getStatement(int i){
- return _statements.get(i);
- }
-
- public ArrayList<Statement> getStatements()
- {
- return _statements;
- }
-
- public void setStatements( ArrayList<Statement> s )
- {
- _statements = s;
- }
-
- public ArrayList<Hop> get_hops() throws HopsException {
- return _hops;
- }
-
- public ArrayList<Lop> getLops() {
- return _lops;
- }
-
- public void set_hops(ArrayList<Hop> hops) {
- _hops = hops;
- }
-
- public void setLops(ArrayList<Lop> lops) {
- _lops = lops;
- }
-
- public boolean mergeable(){
- for (Statement s : _statements){
- if (s.controlStatement())
- return false;
- }
- return true;
- }
-
-
- public boolean isMergeableFunctionCallBlock(DMLProgram dmlProg) throws LanguageException{
-
-// if (DMLScript.ENABLE_DEBUG_MODE && !DMLScript.ENABLE_DEBUG_OPTIMIZER)
- if (DMLScript.ENABLE_DEBUG_MODE)
- return false;
-
- // check whether targetIndex stmt block is for a mergable function call
- Statement stmt = this.getStatement(0);
-
- // Check whether targetIndex block is: control stmt block or stmt block for un-mergable function call
- if ( stmt instanceof WhileStatement || stmt instanceof IfStatement || stmt instanceof ForStatement
- || stmt instanceof FunctionStatement || ( stmt instanceof PrintStatement && ((PrintStatement)stmt).getType() == PRINTTYPE.STOP )/*|| stmt instanceof ELStatement*/ )
- {
- return false;
- }
-
- if (stmt instanceof AssignmentStatement || stmt instanceof MultiAssignmentStatement){
- Expression sourceExpr = null;
- if (stmt instanceof AssignmentStatement) {
- AssignmentStatement astmt = (AssignmentStatement)stmt;
- // for now, ensure that an assignment statement containing a read from csv ends up in own statement block
- if(astmt.getSource().toString().contains(DataExpression.FORMAT_TYPE + "=" + DataExpression.FORMAT_TYPE_VALUE_CSV) && astmt.getSource().toString().contains("read"))
- return false;
- sourceExpr = astmt.getSource();
- }
- else
- sourceExpr = ((MultiAssignmentStatement)stmt).getSource();
- if ( sourceExpr instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression)sourceExpr).multipleReturns() )
- return false;
-
- //function calls (only mergable if inlined dml-bodied function)
- if (sourceExpr instanceof FunctionCallIdentifier){
- FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
- FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
- if (fblock == null){
- LOG.error(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- throw new LanguageException(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- }
- if( !rIsInlineableFunction(fblock, dmlProg) )
- return false;
- }
- }
-
- // regular function block
- return true;
- }
-
- public boolean isRewritableFunctionCall(Statement stmt, DMLProgram dmlProg) throws LanguageException{
-
- // for regular stmt, check if this is a function call stmt block
- if (stmt instanceof AssignmentStatement || stmt instanceof MultiAssignmentStatement){
- Expression sourceExpr = null;
- if (stmt instanceof AssignmentStatement)
- sourceExpr = ((AssignmentStatement)stmt).getSource();
- else
- sourceExpr = ((MultiAssignmentStatement)stmt).getSource();
-
- if (sourceExpr instanceof FunctionCallIdentifier){
- FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
- FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(),fcall.getName());
- if (fblock == null){
- LOG.error(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- throw new LanguageException(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- }
-
- if( rIsInlineableFunction(fblock, dmlProg) )
- return true;
- }
- }
-
- // regular statement
- return false;
- }
-
-
- /**
- *
- * @param fblock
- * @param prog
- * @return
- */
- private boolean rIsInlineableFunction( FunctionStatementBlock fblock, DMLProgram prog )
- {
- boolean ret = true;
-
- //reject external functions and function bodies with multiple blocks
- if( fblock.getStatements().isEmpty() //empty blocks
- || fblock.getStatement(0) instanceof ExternalFunctionStatement
- || ((FunctionStatement)fblock.getStatement(0)).getBody().size() > 1 )
- {
- return false;
- }
-
- //reject control flow and non-inlinable functions
- if(!fblock.getStatements().isEmpty() && !((FunctionStatement)fblock.getStatement(0)).getBody().isEmpty())
- {
- StatementBlock stmtBlock = ((FunctionStatement)fblock.getStatement(0)).getBody().get(0);
-
- //reject control flow blocks
- if (stmtBlock instanceof IfStatementBlock || stmtBlock instanceof WhileStatementBlock || stmtBlock instanceof ForStatementBlock)
- return false;
-
- //recursively check that functions are inlinable
- for( Statement s : stmtBlock.getStatements() ){
- if( s instanceof AssignmentStatement && ((AssignmentStatement)s).getSource() instanceof FunctionCallIdentifier )
- {
- AssignmentStatement as = (AssignmentStatement)s;
- FunctionCallIdentifier fcall = (FunctionCallIdentifier) as.getSource();
- FunctionStatementBlock fblock2 = prog.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
- ret &= rIsInlineableFunction(fblock2, prog);
- if( as.getSource().toString().contains(DataExpression.FORMAT_TYPE + "=" + DataExpression.FORMAT_TYPE_VALUE_CSV) && as.getSource().toString().contains("read"))
- return false;
-
- if( !ret ) return false;
- }
- if( s instanceof MultiAssignmentStatement && ((MultiAssignmentStatement)s).getSource() instanceof FunctionCallIdentifier )
- {
- FunctionCallIdentifier fcall = (FunctionCallIdentifier) ((MultiAssignmentStatement)s).getSource();
- FunctionStatementBlock fblock2 = prog.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
- ret &= rIsInlineableFunction(fblock2, prog);
- if( !ret ) return false;
- }
- }
- }
-
- return ret;
- }
-
- public static ArrayList<StatementBlock> mergeFunctionCalls(ArrayList<StatementBlock> body, DMLProgram dmlProg) throws LanguageException
- {
- for(int i = 0; i <body.size(); i++){
-
- StatementBlock currBlock = body.get(i);
-
- // recurse to children function statement blocks
- if (currBlock instanceof WhileStatementBlock){
- WhileStatement wstmt = (WhileStatement)((WhileStatementBlock)currBlock).getStatement(0);
- wstmt.setBody(mergeFunctionCalls(wstmt.getBody(),dmlProg));
- }
-
- else if (currBlock instanceof ForStatementBlock){
- ForStatement fstmt = (ForStatement)((ForStatementBlock)currBlock).getStatement(0);
- fstmt.setBody(mergeFunctionCalls(fstmt.getBody(),dmlProg));
- }
-
- else if (currBlock instanceof IfStatementBlock){
- IfStatement ifstmt = (IfStatement)((IfStatementBlock)currBlock).getStatement(0);
- ifstmt.setIfBody(mergeFunctionCalls(ifstmt.getIfBody(),dmlProg));
- ifstmt.setElseBody(mergeFunctionCalls(ifstmt.getElseBody(),dmlProg));
- }
-
- else if (currBlock instanceof FunctionStatementBlock){
- FunctionStatement functStmt = (FunctionStatement)((FunctionStatementBlock)currBlock).getStatement(0);
- functStmt.setBody(mergeFunctionCalls(functStmt.getBody(),dmlProg));
- }
- }
-
- ArrayList<StatementBlock> result = new ArrayList<StatementBlock>();
-
- StatementBlock currentBlock = null;
-
- for (int i = 0; i < body.size(); i++){
- StatementBlock current = body.get(i);
- if (current.isMergeableFunctionCallBlock(dmlProg)){
- if (currentBlock != null) {
- currentBlock.addStatementBlock(current);
- } else {
- currentBlock = current;
- }
- } else {
- if (currentBlock != null) {
- result.add(currentBlock);
- }
- result.add(current);
- currentBlock = null;
- }
- }
-
- if (currentBlock != null) {
- result.add(currentBlock);
- }
-
- return result;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append("statements\n");
- for (Statement s : _statements){
- sb.append(s);
- sb.append("\n");
- }
- if (_liveOut != null) sb.append("liveout " + _liveOut.toString() + "\n");
- if (_liveIn!= null) sb.append("livein " + _liveIn.toString()+ "\n");
- if (_gen != null) sb.append("gen " + _gen.toString()+ "\n");
- if (_kill != null) sb.append("kill " + _kill.toString()+ "\n");
- if (_read != null) sb.append("read " + _read.toString()+ "\n");
- if (_updated != null) sb.append("updated " + _updated.toString()+ "\n");
- return sb.toString();
- }
-
- public static ArrayList<StatementBlock> mergeStatementBlocks(ArrayList<StatementBlock> sb){
-
- ArrayList<StatementBlock> result = new ArrayList<StatementBlock>();
-
- if (sb == null || sb.isEmpty()) {
- return new ArrayList<StatementBlock>();
- }
-
- StatementBlock currentBlock = null;
-
- for (int i = 0; i < sb.size(); i++){
- StatementBlock current = sb.get(i);
- if (current.mergeable()){
- if (currentBlock != null) {
- currentBlock.addStatementBlock(current);
- } else {
- currentBlock = current;
- }
- } else {
- if (currentBlock != null) {
- result.add(currentBlock);
- }
- result.add(current);
- currentBlock = null;
- }
- }
-
- if (currentBlock != null) {
- result.add(currentBlock);
- }
-
- return result;
-
- }
-
-
- public ArrayList<Statement> rewriteFunctionCallStatements (DMLProgram dmlProg, ArrayList<Statement> statements) throws LanguageException {
-
- ArrayList<Statement> newStatements = new ArrayList<Statement>();
- for (Statement current : statements){
- if (isRewritableFunctionCall(current, dmlProg)){
-
- Expression sourceExpr = null;
- if (current instanceof AssignmentStatement)
- sourceExpr = ((AssignmentStatement)current).getSource();
- else
- sourceExpr = ((MultiAssignmentStatement)current).getSource();
-
- FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
- FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
- if (fblock == null){
- LOG.error(fcall.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- throw new LanguageException(fcall.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- }
- FunctionStatement fstmt = (FunctionStatement)fblock.getStatement(0);
-
- // recursive inlining (no memo required because update-inplace of function statement blocks, so no redundant inlining)
- if( rIsInlineableFunction(fblock, dmlProg) ){
- fstmt.getBody().get(0).setStatements(rewriteFunctionCallStatements(dmlProg, fstmt.getBody().get(0).getStatements()));
- }
-
- //MB: we cannot use the hash since multiple interleaved inlined functions should be independent.
- //String prefix = new Integer(fblock.hashCode()).toString() + "_";
- String prefix = _seq.getNextID() + "_";
-
- if (fstmt.getBody().size() > 1){
- LOG.error(sourceExpr.printErrorLocation() + "rewritable function can only have 1 statement block");
- throw new LanguageException(sourceExpr.printErrorLocation() + "rewritable function can only have 1 statement block");
- }
- StatementBlock sblock = fstmt.getBody().get(0);
-
- for (int i =0; i < fstmt.getInputParams().size(); i++){
-
- DataIdentifier currFormalParam = fstmt.getInputParams().get(i);
-
- // create new assignment statement
- String newFormalParameterName = prefix + currFormalParam.getName();
- DataIdentifier newTarget = new DataIdentifier(currFormalParam);
- newTarget.setName(newFormalParameterName);
-
- Expression currCallParam = null;
- if (fcall.getParamExprs().size() > i){
- // function call has value for parameter
- currCallParam = fcall.getParamExprs().get(i).getExpr();
- }
- else {
- // use default value for parameter
- if (fstmt.getInputParams().get(i).getDefaultValue() == null){
- LOG.error(currFormalParam.printErrorLocation() + "default parameter for " + currFormalParam + " is undefined");
- throw new LanguageException(currFormalParam.printErrorLocation() + "default parameter for " + currFormalParam + " is undefined");
- }
- currCallParam = new DataIdentifier(fstmt.getInputParams().get(i).getDefaultValue());
- currCallParam.setAllPositions( fstmt.getInputParams().get(i).getFilename(),
- fstmt.getInputParams().get(i).getBeginLine(),
- fstmt.getInputParams().get(i).getBeginColumn(),
- fstmt.getInputParams().get(i).getEndLine(),
- fstmt.getInputParams().get(i).getEndColumn());
- }
-
- //auto casting of inputs on inlining (if required)
- ValueType targetVT = newTarget.getValueType();
- if( newTarget.getDataType()==DataType.SCALAR && currCallParam.getOutput() != null
- && targetVT != currCallParam.getOutput().getValueType() && targetVT != ValueType.STRING )
- {
- currCallParam = new BuiltinFunctionExpression(BuiltinFunctionExpression.getValueTypeCastOperator(targetVT), new Expression[] {currCallParam},
- newTarget.getFilename(), newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn());
- }
-
- // create the assignment statement to bind the call parameter to formal parameter
- AssignmentStatement binding = new AssignmentStatement(newTarget, currCallParam, newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn());
- newStatements.add(binding);
- }
-
- for (Statement stmt : sblock._statements){
-
- // rewrite the statement to use the "rewritten" name
- Statement rewrittenStmt = stmt.rewriteStatement(prefix);
- newStatements.add(rewrittenStmt);
- }
-
- // handle the return values
- for (int i = 0; i < fstmt.getOutputParams().size(); i++){
-
- // get the target (return parameter from function)
- DataIdentifier currReturnParam = fstmt.getOutputParams().get(i);
- String newSourceName = prefix + currReturnParam.getName();
- DataIdentifier newSource = new DataIdentifier(currReturnParam);
- newSource.setName(newSourceName);
-
- // get binding
- DataIdentifier newTarget = null;
- if (current instanceof AssignmentStatement){
- if (i > 0) {
- LOG.error(current.printErrorLocation() + "Assignment statement cannot return multiple values");
- throw new LanguageException(current.printErrorLocation() + "Assignment statement cannot return multiple values");
- }
- newTarget = new DataIdentifier(((AssignmentStatement)current).getTarget());
- }
- else{
- newTarget = new DataIdentifier(((MultiAssignmentStatement)current).getTargetList().get(i));
- }
-
- //auto casting of inputs on inlining (always, redundant cast removed during Hop Rewrites)
- ValueType sourceVT = newSource.getValueType();
- if( newSource.getDataType()==DataType.SCALAR && sourceVT != ValueType.STRING ){
- newSource = new BuiltinFunctionExpression(BuiltinFunctionExpression.getValueTypeCastOperator(sourceVT), new Expression[] {newSource},
- newTarget.getFilename(), newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn());
- }
-
- // create the assignment statement to bind the call parameter to formal parameter
- AssignmentStatement binding = new AssignmentStatement(newTarget, newSource, newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn());
-
- newStatements.add(binding);
- }
-
- } // end if (isRewritableFunctionCall(current, dmlProg)
-
- else {
- newStatements.add(current);
- }
- }
-
- return newStatements;
- }
-
- /**
- *
- * @param dmlProg
- * @param ids
- * @param constVars
- * @param conditional
- * @return
- * @throws LanguageException
- * @throws ParseException
- * @throws IOException
- */
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
- _constVarsIn.putAll(constVars);
- HashMap<String, ConstIdentifier> currConstVars = new HashMap<String,ConstIdentifier>();
- currConstVars.putAll(constVars);
-
- _statements = rewriteFunctionCallStatements(dmlProg, _statements);
- _dmlProg = dmlProg;
-
- for (Statement current : _statements){
-
- if (current instanceof OutputStatement){
- OutputStatement os = (OutputStatement)current;
-
- // validate variable being written by output statement exists
- DataIdentifier target = (DataIdentifier)os.getIdentifier();
- if (ids.getVariable(target.getName()) == null) {
- //undefined variables are always treated unconditionally as error in order to prevent common script-level bugs
- raiseValidateError("Undefined Variable (" + target.getName() + ") used in statement", false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- if ( ids.getVariable(target.getName()).getDataType() == DataType.SCALAR) {
- boolean paramsOkay = true;
- for (String key : os.getSource().getVarParams().keySet()){
- if (! (key.equals(DataExpression.IO_FILENAME) || key.equals(DataExpression.FORMAT_TYPE)))
- paramsOkay = false;
- }
- if( !paramsOkay ) {
- raiseValidateError("Invalid parameters in write statement: " + os.toString(), conditional);
- }
- }
-
- Expression source = os.getSource();
- source.setOutput(target);
- source.validateExpression(ids.getVariables(), currConstVars, conditional);
-
- setStatementFormatType(os, conditional);
- target.setDimensionValueProperties(ids.getVariable(target.getName()));
- }
-
- else if (current instanceof AssignmentStatement){
- AssignmentStatement as = (AssignmentStatement)current;
- DataIdentifier target = as.getTarget();
- Expression source = as.getSource();
-
- if (source instanceof FunctionCallIdentifier) {
- ((FunctionCallIdentifier) source).validateExpression(dmlProg, ids.getVariables(),currConstVars, conditional);
- }
- else {
- if( MLContextProxy.isActive() )
- MLContextProxy.setAppropriateVarsForRead(source, target._name);
-
- source.validateExpression(ids.getVariables(), currConstVars, conditional);
- }
-
- if (source instanceof DataExpression && ((DataExpression)source).getOpCode() == Expression.DataOp.READ)
- setStatementFormatType(as, conditional);
-
- // Handle const vars: (a) basic constant propagation, and (b) transitive constant propagation over assignments
- currConstVars.remove(target.getName());
- if(source instanceof ConstIdentifier && !(target instanceof IndexedIdentifier)){ //basic
- currConstVars.put(target.getName(), (ConstIdentifier)source);
- }
- if( source instanceof DataIdentifier && !(target instanceof IndexedIdentifier) ){ //transitive
- DataIdentifier diSource = (DataIdentifier) source;
- if( currConstVars.containsKey(diSource.getName()) ){
- currConstVars.put(target.getName(), currConstVars.get(diSource.getName()));
- }
- }
-
- if (source instanceof BuiltinFunctionExpression){
- BuiltinFunctionExpression bife = (BuiltinFunctionExpression)source;
- if ( bife.getOpCode() == Expression.BuiltinFunctionOp.NROW
- || bife.getOpCode() == Expression.BuiltinFunctionOp.NCOL )
- {
- DataIdentifier id = (DataIdentifier)bife.getFirstExpr();
- DataIdentifier currVal = ids.getVariable(id.getName());
- if (currVal == null){
- //undefined variables are always treated unconditionally as error in order to prevent common script-level bugs
- bife.raiseValidateError("Undefined Variable (" + id.getName() + ") used in statement", false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- IntIdentifier intid = null;
- if (bife.getOpCode() == Expression.BuiltinFunctionOp.NROW){
- intid = new IntIdentifier(currVal.getDim1(), bife.getFilename(), bife.getBeginLine(), bife.getBeginColumn(), bife.getEndLine(), bife.getEndColumn());
- } else {
- intid = new IntIdentifier(currVal.getDim2(), bife.getFilename(), bife.getBeginLine(), bife.getBeginColumn(), bife.getEndLine(), bife.getEndColumn());
- }
-
- // handle case when nrow / ncol called on variable with size unknown (dims == -1)
- // --> const prop NOT possible
- if (intid.getValue() != -1){
- currConstVars.put(target.getName(), intid);
- }
- }
- }
- // CASE: target NOT indexed identifier
- if (!(target instanceof IndexedIdentifier)){
- target.setProperties(source.getOutput());
- if (source.getOutput() instanceof IndexedIdentifier){
- target.setDimensions(source.getOutput().getDim1(), source.getOutput().getDim2());
- }
-
- }
- // CASE: target is indexed identifier
- else
- {
- // process the "target" being indexed
- DataIdentifier targetAsSeen = ids.getVariable(target.getName());
- if (targetAsSeen == null){
- target.raiseValidateError("cannot assign value to indexed identifier " + target.toString() + " without first initializing " + target.getName(), conditional);
- }
- target.setProperties(targetAsSeen);
-
- // process the expressions for the indexing
- if ( ((IndexedIdentifier)target).getRowLowerBound() != null )
- ((IndexedIdentifier)target).getRowLowerBound().validateExpression(ids.getVariables(), currConstVars, conditional);
- if ( ((IndexedIdentifier)target).getRowUpperBound() != null )
- ((IndexedIdentifier)target).getRowUpperBound().validateExpression(ids.getVariables(), currConstVars, conditional);
- if ( ((IndexedIdentifier)target).getColLowerBound() != null )
- ((IndexedIdentifier)target).getColLowerBound().validateExpression(ids.getVariables(), currConstVars, conditional);
- if ( ((IndexedIdentifier)target).getColUpperBound() != null )
- ((IndexedIdentifier)target).getColUpperBound().validateExpression(ids.getVariables(), currConstVars, conditional);
-
- // validate that LHS indexed identifier is being assigned a matrix value
-// if (source.getOutput().getDataType() != Expression.DataType.MATRIX){
-// LOG.error(target.printErrorLocation() + "Indexed expression " + target.toString() + " can only be assigned matrix value");
-// throw new LanguageException(target.printErrorLocation() + "Indexed expression " + target.toString() + " can only be assigned matrix value");
-// }
-
- // validate that size of LHS index ranges is being assigned:
- // (a) a matrix value of same size as LHS
- // (b) singleton value (semantics: initialize enitre submatrix with this value)
- IndexPair targetSize = ((IndexedIdentifier)target).calculateIndexedDimensions(ids.getVariables(), currConstVars, conditional);
-
- if (targetSize._row >= 1 && source.getOutput().getDim1() > 1 && targetSize._row != source.getOutput().getDim1()){
- target.raiseValidateError("Dimension mismatch. Indexed expression " + target.toString() + " can only be assigned matrix with dimensions "
- + targetSize._row + " rows and " + targetSize._col + " cols. Attempted to assign matrix with dimensions "
- + source.getOutput().getDim1() + " rows and " + source.getOutput().getDim2() + " cols ", conditional);
- }
-
- if (targetSize._col >= 1 && source.getOutput().getDim2() > 1 && targetSize._col != source.getOutput().getDim2()){
- target.raiseValidateError("Dimension mismatch. Indexed expression " + target.toString() + " can only be assigned matrix with dimensions "
- + targetSize._row + " rows and " + targetSize._col + " cols. Attempted to assign matrix with dimensions "
- + source.getOutput().getDim1() + " rows and " + source.getOutput().getDim2() + " cols ", conditional);
- }
-
- ((IndexedIdentifier)target).setDimensions(targetSize._row, targetSize._col);
- }
-
- ids.addVariable(target.getName(), target);
-
- }
-
- else if (current instanceof MultiAssignmentStatement){
- MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
- ArrayList<DataIdentifier> targetList = mas.getTargetList();
-
- // perform validation of source expression
- Expression source = mas.getSource();
- /*
- * MultiAssignmentStatments currently supports only External,
- * User-defined, and Multi-return Builtin function expressions
- */
- if (!(source instanceof DataIdentifier)
- || (source instanceof DataIdentifier && !((DataIdentifier)source).multipleReturns()) ) {
- //if (!(source instanceof FunctionCallIdentifier) ) {
- //|| !(source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression)source).isMultiReturnBuiltinFunction()) ){
- source.raiseValidateError("can only use user-defined functions with multi-assignment statement", conditional);
- }
-
- if ( source instanceof FunctionCallIdentifier) {
- FunctionCallIdentifier fci = (FunctionCallIdentifier)source;
- fci.validateExpression(dmlProg, ids.getVariables(), currConstVars, conditional);
- }
- else if ( source instanceof BuiltinFunctionExpression && ((DataIdentifier)source).multipleReturns()) {
- source.validateExpression(mas, ids.getVariables(), currConstVars, conditional);
- }
- else
- throw new LanguageException("Unexpected error.");
-
-
- if ( source instanceof FunctionCallIdentifier ) {
- for (int j =0; j< targetList.size(); j++){
-
- DataIdentifier target = targetList.get(j);
- // set target properties (based on type info in function call statement return params)
- FunctionCallIdentifier fci = (FunctionCallIdentifier)source;
- FunctionStatement fstmt = (FunctionStatement)_dmlProg.getFunctionStatementBlock(fci.getNamespace(), fci.getName()).getStatement(0);
- if (fstmt == null){
- fci.raiseValidateError(" function " + fci.getName() + " is undefined in namespace " + fci.getNamespace(), conditional);
- }
- if (!(target instanceof IndexedIdentifier)){
- target.setProperties(fstmt.getOutputParams().get(j));
- }
- else{
- DataIdentifier targetAsSeen = ids.getVariable(target.getName());
- if (targetAsSeen == null){
- raiseValidateError(target.printErrorLocation() + "cannot assign value to indexed identifier " + target.toString() + " without first initializing " + target.getName(), conditional);
- }
- target.setProperties(targetAsSeen);
- }
- ids.addVariable(target.getName(), target);
- }
- }
- else if ( source instanceof BuiltinFunctionExpression ) {
- Identifier[] outputs = source.getOutputs();
- for (int j=0; j < targetList.size(); j++) {
- ids.addVariable(targetList.get(j).getName(), (DataIdentifier)outputs[j]);
- }
- }
- }
-
- else if(current instanceof ForStatement || current instanceof IfStatement || current instanceof WhileStatement ){
- raiseValidateError("control statement (CVStatement, ELStatement, WhileStatement, IfStatement, ForStatement) should not be in genreric statement block. Likely a parsing error", conditional);
- }
-
- else if (current instanceof PrintStatement){
- PrintStatement pstmt = (PrintStatement) current;
- Expression expr = pstmt.getExpression();
- expr.validateExpression(ids.getVariables(), currConstVars, conditional);
-
- // check that variables referenced in print statement expression are scalars
- if (expr.getOutput().getDataType() != Expression.DataType.SCALAR){
- raiseValidateError("print statement can only print scalars", conditional);
- }
- }
-
- // no work to perform for PathStatement or ImportStatement
- else if (current instanceof PathStatement){}
- else if (current instanceof ImportStatement){}
-
-
- else {
- raiseValidateError("cannot process statement of type " + current.getClass().getSimpleName(), conditional);
- }
-
- } // end for (Statement current : _statements){
- _constVarsOut.putAll(currConstVars);
- return ids;
-
- }
-
- public void setStatementFormatType(OutputStatement s, boolean conditionalValidate)
- throws LanguageException, ParseException
- {
- //case of specified format parameter
- if (s.getExprParam(DataExpression.FORMAT_TYPE)!= null )
- {
- Expression formatTypeExpr = s.getExprParam(DataExpression.FORMAT_TYPE);
- if (!(formatTypeExpr instanceof StringIdentifier)){
- raiseValidateError("IO statement parameter " + DataExpression.FORMAT_TYPE
- + " can only be a string with one of following values: binary, text, mm, csv.", false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- String ft = formatTypeExpr.toString();
- if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_BINARY)){
- s.getIdentifier().setFormatType(FormatType.BINARY);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_TEXT)){
- s.getIdentifier().setFormatType(FormatType.TEXT);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET)){
- s.getIdentifier().setFormatType(FormatType.MM);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_CSV)){
- s.getIdentifier().setFormatType(FormatType.CSV);
- } else{
- raiseValidateError("IO statement parameter " + DataExpression.FORMAT_TYPE
- + " can only be a string with one of following values: binary, text, mm, csv; invalid format: '"+ft+"'.", false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- //case of unspecified format parameter, use default
- else
- {
- s.addExprParam(DataExpression.FORMAT_TYPE, new StringIdentifier(FormatType.TEXT.toString(),
- s.getFilename(), s.getBeginLine(), s.getBeginColumn(), s.getEndLine(), s.getEndColumn()), true);
- s.getIdentifier().setFormatType(FormatType.TEXT);
- }
- }
-
- public void setStatementFormatType(AssignmentStatement s, boolean conditionalValidate)
- throws LanguageException, ParseException
- {
-
- if (!(s.getSource() instanceof DataExpression))
- return;
- DataExpression dataExpr = (DataExpression)s.getSource();
-
- if (dataExpr.getVarParam(DataExpression.FORMAT_TYPE)!= null ){
-
- Expression formatTypeExpr = dataExpr.getVarParam(DataExpression.FORMAT_TYPE);
- if (!(formatTypeExpr instanceof StringIdentifier)){
- raiseValidateError("IO statement parameter " + DataExpression.FORMAT_TYPE
- + " can only be a string with one of following values: binary, text", conditionalValidate, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- String ft = formatTypeExpr.toString();
- if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_BINARY)){
- s.getTarget().setFormatType(FormatType.BINARY);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_TEXT)){
- s.getTarget().setFormatType(FormatType.TEXT);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET)){
- s.getTarget().setFormatType(FormatType.MM);
- } else if (ft.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_CSV)){
- s.getTarget().setFormatType(FormatType.CSV);
- } else{
- raiseValidateError("IO statement parameter " + DataExpression.FORMAT_TYPE
- + " can only be a string with one of following values: binary, text, mm, csv", conditionalValidate, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- } else {
- dataExpr.addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(FormatType.TEXT.toString(),
- dataExpr.getFilename(), dataExpr.getBeginLine(), dataExpr.getBeginColumn(), dataExpr.getEndLine(), dataExpr.getEndColumn()));
- s.getTarget().setFormatType(FormatType.TEXT);
- }
- }
-
-
- /**
- * For each statement:
- *
- * gen rule: for each variable read in current statement but not updated in any PRIOR statement, add to gen
- * Handles case where variable both read and updated in same statement (i = i + 1, i needs to be added to gen)
- *
- * kill rule: for each variable updated in current statement but not read in this or any PRIOR statement,
- * add to kill.
- *
- */
- @Override
- public VariableSet initializeforwardLV(VariableSet activeIn) throws LanguageException {
-
- for (Statement s : _statements){
- s.initializeforwardLV(activeIn);
- VariableSet read = s.variablesRead();
- VariableSet updated = s.variablesUpdated();
-
- if (s instanceof WhileStatement || s instanceof IfStatement || s instanceof ForStatement){
- LOG.error(s.printErrorLocation() + "control statement (while / for / if) cannot be in generic statement block");
- throw new LanguageException(s.printErrorLocation() + "control statement (while / for / if) cannot be in generic statement block");
- }
-
- if (read != null){
- // for each variable read in this statement but not updated in
- // any prior statement, add to sb._gen
-
- for (String var : read.getVariableNames()) {
- if (!_updated.containsVariable(var)) {
- _gen.addVariable(var, read.getVariable(var));
- }
- }
- }
-
- _read.addVariables(read);
- _updated.addVariables(updated);
-
- if (updated != null) {
- // for each updated variable that is not read
- for (String var : updated.getVariableNames())
- {
- //NOTE MB: always add updated vars to kill (in order to prevent side effects
- //of implicitly updated statistics over common data identifiers, propagated from
- //downstream operators to its inputs due to 'livein = gen \cup (liveout-kill))'.
- _kill.addVariable(var, _updated.getVariable(var));
-
- //if (!_read.containsVariable(var)) {
- // _kill.addVariable(var, _updated.getVariable(var));
- //}
- }
- }
- }
- _liveOut = new VariableSet();
- _liveOut.addVariables(activeIn);
- _liveOut.addVariables(_updated);
- return _liveOut;
- }
-
- @Override
- public VariableSet initializebackwardLV(VariableSet loPassed)
- throws LanguageException
- {
- int numStatements = _statements.size();
- VariableSet lo = new VariableSet(loPassed);
- for (int i = numStatements-1; i>=0; i--){
- lo = _statements.get(i).initializebackwardLV(lo);
- }
-
- return new VariableSet(lo);
- }
-
- public HashMap<String, ConstIdentifier> getConstIn(){
- return _constVarsIn;
- }
-
- public HashMap<String, ConstIdentifier> getConstOut(){
- return _constVarsOut;
- }
-
-
- public VariableSet analyze(VariableSet loPassed)
- throws LanguageException{
-
- VariableSet candidateLO = new VariableSet();
- candidateLO.addVariables(loPassed);
- //candidateLO.addVariables(_gen);
-
- VariableSet origLiveOut = new VariableSet();
- origLiveOut.addVariables(_liveOut);
-
- _liveOut = new VariableSet();
- for (String name : candidateLO.getVariableNames()){
- if (origLiveOut.containsVariable(name)){
- _liveOut.addVariable(name, candidateLO.getVariable(name));
- }
- }
-
- initializebackwardLV(_liveOut);
-
- _liveIn = new VariableSet();
- _liveIn.addVariables(_liveOut);
- _liveIn.removeVariables(_kill);
- _liveIn.addVariables(_gen);
-
- VariableSet liveInReturn = new VariableSet();
- liveInReturn.addVariables(_liveIn);
- return liveInReturn;
- }
-
- ///////////////////////////////////////////////////////////////
- // validate error handling (consistent for all expressions)
-
- /**
- *
- * @param msg
- * @param conditional
- * @throws LanguageException
- */
- public void raiseValidateError( String msg, boolean conditional )
- throws LanguageException
- {
- raiseValidateError(msg, conditional, null);
- }
-
- /**
- *
- * @param msg
- * @param conditional
- * @param code
- * @throws LanguageException
- */
- public void raiseValidateError( String msg, boolean conditional, String errorCode )
- throws LanguageException
- {
- if( conditional ) //warning if conditional
- {
- String fullMsg = this.printWarningLocation() + msg;
-
- LOG.warn( fullMsg );
- }
- else //error and exception if unconditional
- {
- String fullMsg = this.printErrorLocation() + msg;
-
- LOG.error( fullMsg );
- if( errorCode != null )
- throw new LanguageException( fullMsg, errorCode );
- else
- throw new LanguageException( fullMsg );
- }
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for statement blocks
- ///////////////////////////////////////////////////////////////////////////
- private String _filename = "MAIN SCRIPT";
- private int _beginLine = 0, _beginColumn = 0;
- private int _endLine = 0, _endColumn = 0;
-
- public void setFilename (String fname) { _filename = fname; }
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(String fname, int blp, int bcp, int elp, int ecp){
- _filename = fname;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public String getFilename() { return _filename; }
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: " + _filename + " -- statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
-
- /////////
- // materialized hops recompilation flags
- ////
-
- public void updateRecompilationFlag()
- throws HopsException
- {
- _requiresRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && Recompiler.requiresRecompilation(get_hops());
- }
-
- public boolean requiresRecompilation()
- {
- return _requiresRecompile;
- }
-
-
-} // end class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/StringIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/StringIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/StringIdentifier.java
deleted file mode 100644
index 260cfe6..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/StringIdentifier.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class StringIdentifier extends ConstIdentifier
-{
-
- private String _val;
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- return this;
- }
-
- public StringIdentifier(String val, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = val;
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.STRING);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
-
- }
-
- public StringIdentifier(StringIdentifier s){
- super();
- _val = s.getValue();
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.STRING);
- }
-
- public String getValue(){
- return _val;
- }
-
- public String toString(){
- return _val;
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public long getLongValue()
- throws LanguageException
- {
- throw new LanguageException("Unsupported string-to-long conversion.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/Token.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/Token.java b/src/main/java/com/ibm/bi/dml/parser/Token.java
deleted file mode 100644
index a0b0bef..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/Token.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
-/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package com.ibm.bi.dml.parser;
-
-/**
- * Describes the input token stream.
- */
-
-public class Token implements java.io.Serializable {
-
- /**
- * The version identifier for this Serializable class.
- * Increment only if the <i>serialized</i> form of the
- * class changes.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * An integer that describes the kind of this token. This numbering
- * system is determined by JavaCCParser, and a table of these numbers is
- * stored in the file ...Constants.java.
- */
- public int kind;
-
- /** The line number of the first character of this Token. */
- public int beginLine;
- /** The column number of the first character of this Token. */
- public int beginColumn;
- /** The line number of the last character of this Token. */
- public int endLine;
- /** The column number of the last character of this Token. */
- public int endColumn;
-
- /**
- * The string image of the token.
- */
- public String image;
-
- /**
- * A reference to the next regular (non-special) token from the input
- * stream. If this is the last token from the input stream, or if the
- * token manager has not read tokens beyond this one, this field is
- * set to null. This is true only if this token is also a regular
- * token. Otherwise, see below for a description of the contents of
- * this field.
- */
- public Token next;
-
- /**
- * This field is used to access special tokens that occur prior to this
- * token, but after the immediately preceding regular (non-special) token.
- * If there are no such special tokens, this field is set to null.
- * When there are more than one such special token, this field refers
- * to the last of these special tokens, which in turn refers to the next
- * previous special token through its specialToken field, and so on
- * until the first special token (whose specialToken field is null).
- * The next fields of special tokens refer to other special tokens that
- * immediately follow it (without an intervening regular token). If there
- * is no such token, this field is null.
- */
- public Token specialToken;
-
- /**
- * An optional attribute value of the Token.
- * Tokens which are not used as syntactic sugar will often contain
- * meaningful values that will be used later on by the compiler or
- * interpreter. This attribute value is often different from the image.
- * Any subclass of Token that actually wants to return a non-null value can
- * override this method as appropriate.
- */
- public Object getValue() {
- return null;
- }
-
- /**
- * No-argument constructor
- */
- public Token() {}
-
- /**
- * Constructs a new token for the specified Image.
- */
- public Token(int kind)
- {
- this(kind, null);
- }
-
- /**
- * Constructs a new token for the specified Image and Kind.
- */
- public Token(int kind, String image)
- {
- this.kind = kind;
- this.image = image;
- }
-
- /**
- * Returns the image.
- */
- public String toString()
- {
- return image;
- }
-
- /**
- * Returns a new Token object, by default. However, if you want, you
- * can create and return subclass objects based on the value of ofKind.
- * Simply add the cases to the switch for all those special cases.
- * For example, if you have a subclass of Token called IDToken that
- * you want to create if ofKind is ID, simply add something like :
- *
- * case MyParserConstants.ID : return new IDToken(ofKind, image);
- *
- * to the following switch statement. Then you can cast matchedToken
- * variable to the appropriate type and use sit in your lexical actions.
- */
- public static Token newToken(int ofKind, String image)
- {
- switch(ofKind)
- {
- default : return new Token(ofKind, image);
- }
- }
-
- public static Token newToken(int ofKind)
- {
- return newToken(ofKind, null);
- }
-
-}
-/* JavaCC - OriginalChecksum=c5b6b2a4049a8b8df3f1f477aabf47d6 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/VariableSet.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/VariableSet.java b/src/main/java/com/ibm/bi/dml/parser/VariableSet.java
deleted file mode 100644
index f99ca5f..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/VariableSet.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-import java.util.Set;
-
-public class VariableSet
-{
-
- private HashMap<String,DataIdentifier> _variables;
-
- public VariableSet(){
- _variables = new HashMap<String,DataIdentifier>();
- }
-
- public VariableSet( VariableSet vs )
- {
- _variables = new HashMap<String,DataIdentifier>();
-
- if (vs != null) {
- HashMap<String,DataIdentifier> vars = vs.getVariables();
- _variables.putAll(vars);
- }
- }
-
- public void addVariable(String name, DataIdentifier id)
- {
- _variables.put(name,id);
- }
-
- public void addVariables(VariableSet vs)
- {
- if (vs != null) {
- HashMap<String,DataIdentifier> vars = vs.getVariables();
- _variables.putAll(vars);
- }
- }
-
- public void removeVariables(VariableSet vs)
- {
- if( vs != null ){
- Set<String> vars = vs.getVariables().keySet();
- for (String var : vars)
- _variables.remove(var);
- }
- }
-
- public void removeVariable(String name)
- {
- _variables.remove(name);
- }
-
- public boolean containsVariable(String name){
- return _variables.containsKey(name);
- }
-
- public DataIdentifier getVariable(String name){
- return _variables.get(name);
- }
-
- public Set<String> getVariableNames(){
- return _variables.keySet();
- }
-
- public HashMap<String,DataIdentifier> getVariables(){
- return _variables;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- for (String var : _variables.keySet()){
- sb.append(var + ",");
- }
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/WhileStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/WhileStatement.java b/src/main/java/com/ibm/bi/dml/parser/WhileStatement.java
deleted file mode 100644
index 740b31b..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/WhileStatement.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-
-
-public class WhileStatement extends Statement
-{
-
-
- private ConditionalPredicate _predicate;
- private ArrayList<StatementBlock> _body;
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should not call rewriteStatement for WhileStatement");
- throw new LanguageException(this.printErrorLocation() + "should not call rewriteStatement for WhileStatement");
- }
-
- public WhileStatement(){
- _predicate = null;
- _body = new ArrayList<StatementBlock>();
- }
-
- public void setPredicate(ConditionalPredicate pred){
- _predicate = pred;
- }
-
-
- public void addStatementBlock(StatementBlock sb){
- _body.add(sb);
- }
-
- public ConditionalPredicate getConditionalPredicate(){
- return _predicate;
- }
-
- public ArrayList<StatementBlock> getBody(){
- return _body;
- }
-
- public void setBody(ArrayList<StatementBlock> body){
- _body = body;
- }
-
- @Override
- public boolean controlStatement() {
- return true;
- }
-
-
- public void mergeStatementBlocks(){
- _body = StatementBlock.mergeStatementBlocks(_body);
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append("while ( ");
- sb.append(_predicate.toString());
- sb.append(") { \n");
- for (StatementBlock block : _body){
- sb.append(block.toString());
- }
- sb.append("}\n");
- return sb.toString();
- }
-
- public void initializeforwardLV(VariableSet activeIn) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement");
- }
-
- public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement");
-
- }
-
- @Override
- public VariableSet variablesRead() {
- LOG.warn(this.printWarningLocation() + "should not call variablesRead from WhileStatement ");
- return new VariableSet();
- }
-
- @Override
- public VariableSet variablesUpdated() {
- LOG.warn(this.printWarningLocation() + "should not call variablesRead from WhileStatement ");
- return new VariableSet();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/WhileStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/WhileStatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/WhileStatementBlock.java
deleted file mode 100644
index 679fbee..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/WhileStatementBlock.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-
-
-public class WhileStatementBlock extends StatementBlock
-{
-
- private Hop _predicateHops;
- private Lop _predicateLops = null;
- private boolean _requiresPredicateRecompile = false;
-
- @Override
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
- if (_statements.size() > 1){
- raiseValidateError("WhileStatementBlock should have only 1 statement (while statement)", conditional);
- }
-
- WhileStatement wstmt = (WhileStatement) _statements.get(0);
- ConditionalPredicate predicate = wstmt.getConditionalPredicate();
-
- // Record original size information before loop for ALL variables
- // Will compare size / type info for these after loop completes
- // Replace variables with changed size with unknown value
- VariableSet origVarsBeforeBody = new VariableSet();
- for (String key : ids.getVariableNames()){
- DataIdentifier origId = ids.getVariable(key);
- DataIdentifier copyId = new DataIdentifier(origId);
- origVarsBeforeBody.addVariable(key, copyId);
- }
-
- //////////////////////////////////////////////////////////////////////////////
- // FIRST PASS: process the predicate / statement blocks in the body of the for statement
- ///////////////////////////////////////////////////////////////////////////////
-
- //remove updated vars from constants
- for( String var : _updated.getVariableNames() )
- if( constVars.containsKey( var ) )
- constVars.remove( var );
-
- // process the statement blocks in the body of the while statement
- predicate.getPredicate().validateExpression(ids.getVariables(), constVars, conditional);
- ArrayList<StatementBlock> body = wstmt.getBody();
-
- _dmlProg = dmlProg;
- for(StatementBlock sb : body)
- {
- //always conditional
- ids = sb.validate(dmlProg, ids, constVars, true);
- constVars = sb.getConstOut();
- }
-
- if (!body.isEmpty()) {
- _constVarsIn.putAll(body.get(0).getConstIn());
- _constVarsOut.putAll(body.get(body.size()-1).getConstOut());
- }
-
- // for each updated variable
- boolean revalidationRequired = false;
- for (String key : _updated.getVariableNames())
- {
- DataIdentifier startVersion = origVarsBeforeBody.getVariable(key);
- DataIdentifier endVersion = ids.getVariable(key);
-
- if (startVersion != null && endVersion != null)
- {
- //handle data type change (reject)
- if (!startVersion.getOutput().getDataType().equals(endVersion.getOutput().getDataType())){
- raiseValidateError("WhileStatementBlock has unsupported conditional data type change of variable '"+key+"' in loop body.", conditional);
- }
-
- //handle size change
- long startVersionDim1 = (startVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)startVersion).getOrigDim1() : startVersion.getDim1();
- long endVersionDim1 = (endVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)endVersion).getOrigDim1() : endVersion.getDim1();
- long startVersionDim2 = (startVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)startVersion).getOrigDim2() : startVersion.getDim2();
- long endVersionDim2 = (endVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)endVersion).getOrigDim2() : endVersion.getDim2();
-
- boolean sizeUnchanged = ((startVersionDim1 == endVersionDim1) &&
- (startVersionDim2 == endVersionDim2) );
-
- //handle sparsity change
- //NOTE: nnz not propagated via validate, and hence, we conservatively assume that nnz have been changed.
- //long startVersionNNZ = startVersion.getNnz();
- //long endVersionNNZ = endVersion.getNnz();
- //boolean nnzUnchanged = (startVersionNNZ == endVersionNNZ);
- boolean nnzUnchanged = false;
-
- // IF size has changed --
- if (!sizeUnchanged || !nnzUnchanged){
- revalidationRequired = true;
- DataIdentifier recVersion = new DataIdentifier(endVersion);
- if(!sizeUnchanged)
- recVersion.setDimensions(-1, -1);
- if(!nnzUnchanged)
- recVersion.setNnz(-1);
- origVarsBeforeBody.addVariable(key, recVersion);
- }
- }
- }
-
-
- // revalidation is required -- size was updated for at least 1 variable
- if (revalidationRequired)
- {
- // update ids to the reconciled values
- ids = origVarsBeforeBody;
-
- //////////////////////////////////////////////////////////////////////////////
- // SECOND PASS: process the predicate / statement blocks in the body of the for statement
- ///////////////////////////////////////////////////////////////////////////////
-
- //remove updated vars from constants
- for( String var : _updated.getVariableNames() )
- if( constVars.containsKey( var ) )
- constVars.remove( var );
-
- // process the statement blocks in the body of the while statement
- predicate.getPredicate().validateExpression(ids.getVariables(), constVars, conditional);
- body = wstmt.getBody();
-
- _dmlProg = dmlProg;
- for(StatementBlock sb : body)
- {
- //always conditional
- ids = sb.validate(dmlProg, ids, constVars, true);
- constVars = sb.getConstOut();
- }
-
- if (!body.isEmpty()) {
- _constVarsIn.putAll(body.get(0).getConstIn());
- _constVarsOut.putAll(body.get(body.size()-1).getConstOut());
- }
- }
-
- return ids;
- }
-
-
- public VariableSet initializeforwardLV(VariableSet activeInPassed) throws LanguageException {
-
- WhileStatement wstmt = (WhileStatement)_statements.get(0);
- if (_statements.size() > 1){
- LOG.error(_statements.get(0).printErrorLocation() + "WhileStatementBlock should have only 1 statement (while statement)");
- throw new LanguageException(_statements.get(0).printErrorLocation() + "WhileStatementBlock should have only 1 statement (while statement)");
- }
-
- _read = new VariableSet();
- _read.addVariables(wstmt.getConditionalPredicate().variablesRead());
- _updated.addVariables(wstmt.getConditionalPredicate().variablesUpdated());
-
- _gen = new VariableSet();
- _gen.addVariables(wstmt.getConditionalPredicate().variablesRead());
-
- VariableSet current = new VariableSet();
- current.addVariables(activeInPassed);
-
- for( StatementBlock sb : wstmt.getBody() )
- {
- current = sb.initializeforwardLV(current);
-
- // for each generated variable in this block, check variable not killed
- // in prior statement block in while stmt blody
- for (String varName : sb._gen.getVariableNames()){
-
- // IF the variable is NOT set in the while loop PRIOR to this stmt block,
- // THEN needs to be generated
- if (!_kill.getVariableNames().contains(varName)){
- _gen.addVariable(varName, sb._gen.getVariable(varName));
- }
- }
-
- _read.addVariables(sb._read);
- _updated.addVariables(sb._updated);
-
- // only add kill variables for statement blocks guaranteed to execute
- if (!(sb instanceof WhileStatementBlock) && !(sb instanceof ForStatementBlock) ){
- _kill.addVariables(sb._kill);
- }
- }
-
- // set preliminary "warn" set -- variables that if used later may cause runtime error
- // if the loop is not executed
- // warnSet = (updated MINUS (updatedIfBody INTERSECT updatedElseBody)) MINUS current
- for (String varName : _updated.getVariableNames()){
- if (!activeInPassed.containsVariable(varName)) {
- _warnSet.addVariable(varName, _updated.getVariable(varName));
- }
- }
-
- // activeOut includes variables from passed live in and updated in the while body
- _liveOut = new VariableSet();
- _liveOut.addVariables(current);
- _liveOut.addVariables(_updated);
- return _liveOut;
- }
-
- public VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException{
-
- WhileStatement wstmt = (WhileStatement)_statements.get(0);
-
- VariableSet lo = new VariableSet();
- lo.addVariables(loPassed);
-
- // calls analyze for each statement block in while stmt body
- int numBlocks = wstmt.getBody().size();
- for (int i = numBlocks - 1; i >= 0; i--){
- lo = wstmt.getBody().get(i).analyze(lo);
- }
-
- VariableSet loReturn = new VariableSet();
- loReturn.addVariables(lo);
- return loReturn;
-
- }
-
- public void setPredicateHops(Hop hops) {
- _predicateHops = hops;
- }
-
- public ArrayList<Hop> get_hops() throws HopsException {
-
- if (_hops != null && !_hops.isEmpty()){
- LOG.error(this._statements.get(0).printErrorLocation() + "there should be no HOPs associated with the WhileStatementBlock");
- throw new HopsException(this._statements.get(0).printErrorLocation() + "there should be no HOPs associated with the WhileStatementBlock");
- }
-
- return _hops;
- }
-
- public Hop getPredicateHops(){
- return _predicateHops;
- }
-
- public Lop get_predicateLops() {
- return _predicateLops;
- }
-
- public void set_predicateLops(Lop predicateLops) {
- _predicateLops = predicateLops;
- }
-
- public VariableSet analyze(VariableSet loPassed) throws LanguageException{
-
- VariableSet predVars = new VariableSet();
- predVars.addVariables(((WhileStatement)_statements.get(0)).getConditionalPredicate().variablesRead());
- predVars.addVariables(((WhileStatement)_statements.get(0)).getConditionalPredicate().variablesUpdated());
-
- VariableSet candidateLO = new VariableSet();
- candidateLO.addVariables(loPassed);
- candidateLO.addVariables(_gen);
- candidateLO.addVariables(predVars);
-
- VariableSet origLiveOut = new VariableSet();
- origLiveOut.addVariables(_liveOut);
- origLiveOut.addVariables(predVars);
- origLiveOut.addVariables(_gen);
-
- _liveOut = new VariableSet();
- for (String name : candidateLO.getVariableNames()){
- if (origLiveOut.containsVariable(name)){
- _liveOut.addVariable(name, candidateLO.getVariable(name));
- }
- }
-
- initializebackwardLV(_liveOut);
-
- // set final warnSet: remove variables NOT in live out
- VariableSet finalWarnSet = new VariableSet();
- for (String varName : _warnSet.getVariableNames()){
- if (_liveOut.containsVariable(varName)){
- finalWarnSet.addVariable(varName,_warnSet.getVariable(varName));
- }
- }
- _warnSet = finalWarnSet;
-
- // for now just print the warn set
- for (String varName : _warnSet.getVariableNames()){
- LOG.warn(_warnSet.getVariable(varName).printWarningLocation() + "Initialization of " + varName + " depends on while execution");
- }
-
- // Cannot remove kill variables
- _liveIn = new VariableSet();
- _liveIn.addVariables(_liveOut);
- _liveIn.addVariables(_gen);
-
- VariableSet liveInReturn = new VariableSet();
- liveInReturn.addVariables(_liveIn);
-
- return liveInReturn;
- }
-
- /////////
- // materialized hops recompilation flags
- ////
-
- public void updatePredicateRecompilationFlag()
- throws HopsException
- {
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && Recompiler.requiresRecompilation(getPredicateHops());
- }
-
- public boolean requiresPredicateRecompilation()
- {
- return _requiresPredicateRecompile;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/DMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/DMLParserWrapper.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/DMLParserWrapper.java
deleted file mode 100644
index 5cf6cdd..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/DMLParserWrapper.java
+++ /dev/null
@@ -1,384 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.antlr.v4.runtime.BailErrorStrategy;
-import org.antlr.v4.runtime.DefaultErrorStrategy;
-import org.antlr.v4.runtime.atn.PredictionMode;
-import org.antlr.v4.runtime.misc.ParseCancellationException;
-import org.antlr.v4.runtime.tree.ParseTreeWalker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DmlprogramContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-/**
- * This is the main entry point for the Antlr4 parser.
- * Dml.g4 is the grammar file which enforces syntactic structure of DML program.
- * DmlSyntaticValidator on other hand captures little bit of semantic as well as does the job of translation of Antlr AST to DMLProgram.
- * At a high-level, DmlSyntaticValidator implements call-back methods that are called by walker.walk(validator, tree)
- * The callback methods are of two type: enterSomeASTNode() and exitSomeASTNode()
- * It is important to note that almost every node in AST has either ExpressionInfo or StatementInfo object associated with it.
- * The key design decision is that while "exiting" the node (i.e. callback to exitSomeASTNode), we use information in given AST node and construct an object of type Statement or Expression and put it in StatementInfo or ExpressionInfo respectively.
- * This way it avoids any bugs due to lookahead and one only has to "think as an AST node", thereby making any changes to parse code much simpler :)
- *
- * Note: to add additional builtin function, one only needs to modify DmlSyntaticValidator (which is java file and provides full Eclipse tooling support) not g4.
- *
- * To separate logic of semantic validation, DmlSyntaticValidatorHelper contains functions that do semantic validation. Currently, there is no semantic validation as most of it is delegated to subsequent validation phase.
- *
- * Whenever there is a parse error, it goes through DmlSyntacticErrorListener. This allows us to pipe the error messages to any future pipeline as well as control the format in an elegant manner.
- * There are three types of messages passed:
- * - Syntactic errors: When passed DML script doesnot conform to syntatic structure enforced by Dml.g4
- * - Validation errors: Errors due to translation of AST to DMLProgram
- * - Validation warnings: Messages to inform users that there might be potential bug in their program
- *
- * As of this moment, Antlr4ParserWrapper is stateful and cannot be multithreaded. This is not big deal because each users calls SystemML in different process.
- * If in future we intend to make it multi-threaded, look at cleanUpState method and resolve the dependency accordingly.
- *
- */
-public class DMLParserWrapper extends AParserWrapper
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- /**
- * Custom wrapper to convert statement into statement blocks. Called by doParse and in DmlSyntacticValidator for for, parfor, while, ...
- * @param current a statement
- * @return corresponding statement block
- */
- public static StatementBlock getStatementBlock(Statement current) {
- StatementBlock blk = null;
- if(current instanceof ParForStatement) {
- blk = new ParForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof ForStatement) {
- blk = new ForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof IfStatement) {
- blk = new IfStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof WhileStatement) {
- blk = new WhileStatementBlock();
- blk.addStatement(current);
- }
- else {
- // This includes ImportStatement
- blk = new StatementBlock();
- blk.addStatement(current);
- }
- return blk;
- }
-
- /**
- * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
- * @param fileName either full path or null --> only used for better error handling
- * @param dmlScript required
- * @param argVals
- * @return
- * @throws ParseException
- */
- @Override
- public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
- throws ParseException
- {
- DMLProgram prog = null;
-
- if(dmlScript == null || dmlScript.trim().isEmpty()) {
- throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
- }
-
- // Set the pipeline required for ANTLR parsing
- DMLParserWrapper parser = new DMLParserWrapper();
- prog = parser.doParse(fileName, dmlScript, argVals);
-
- if(prog == null) {
- throw new ParseException("One or more errors found during parsing (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
- }
- return prog;
- }
-
- /**
- * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
- * @param fileName
- * @return null if atleast one error
- */
- public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram dmlPgm = null;
-
- org.antlr.v4.runtime.ANTLRInputStream in;
- try {
- if(dmlScript == null) {
- dmlScript = readDMLScript(fileName);
- }
-
- InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
- in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
-// else {
-// if(!(new File(fileName)).exists()) {
-// throw new ParseException("ERROR: Cannot open file:" + fileName);
-// }
-// in = new org.antlr.v4.runtime.ANTLRInputStream(new java.io.FileInputStream(fileName));
-// }
- } catch (FileNotFoundException e) {
- throw new ParseException("ERROR: Cannot find file:" + fileName);
- } catch (IOException e) {
- throw new ParseException("ERROR: Cannot open file:" + fileName);
- } catch (LanguageException e) {
- throw new ParseException("ERROR: " + e.getMessage());
- }
-
- DmlprogramContext ast = null;
- CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
-
- try {
- DmlLexer lexer = new DmlLexer(in);
- org.antlr.v4.runtime.CommonTokenStream tokens = new org.antlr.v4.runtime.CommonTokenStream(lexer);
- DmlParser antlr4Parser = new DmlParser(tokens);
-
- boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
-
- if(tryOptimizedParsing) {
- // Try faster and simpler SLL
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
- antlr4Parser.removeErrorListeners();
- antlr4Parser.setErrorHandler(new BailErrorStrategy());
- try{
- ast = antlr4Parser.dmlprogram();
- // If successful, no need to try out full LL(*) ... SLL was enough
- }
- catch(ParseCancellationException ex) {
- // Error occurred, so now try full LL(*) for better error messages
- tokens.reset();
- antlr4Parser.reset();
- if(fileName != null) {
- errorListener.pushCurrentFileName(fileName);
- // DmlSyntacticErrorListener.currentFileName.push(fileName);
- }
- else {
- errorListener.pushCurrentFileName("MAIN_SCRIPT");
- // DmlSyntacticErrorListener.currentFileName.push("MAIN_SCRIPT");
- }
- // Set our custom error listener
- antlr4Parser.addErrorListener(errorListener);
- antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
- ast = antlr4Parser.dmlprogram();
- }
- }
- else {
- // Set our custom error listener
- antlr4Parser.removeErrorListeners();
- antlr4Parser.addErrorListener(errorListener);
- errorListener.pushCurrentFileName(fileName);
-
- // Now do the parsing
- ast = antlr4Parser.dmlprogram();
- }
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot parse the program:" + fileName);
- }
-
-
- try {
- // Now convert the parse tree into DMLProgram
- // Do syntactic validation while converting
- org.antlr.v4.runtime.tree.ParseTree tree = ast;
- // And also do syntactic validation
- org.antlr.v4.runtime.tree.ParseTreeWalker walker = new ParseTreeWalker();
- DmlSyntacticValidatorHelper helper = new DmlSyntacticValidatorHelper(errorListener);
- DmlSyntacticValidator validator = new DmlSyntacticValidator(helper, errorListener.peekFileName(), argVals);
- walker.walk(validator, tree);
- errorListener.popFileName();
- if(errorListener.isAtleastOneError()) {
- return null;
- }
- dmlPgm = createDMLProgram(ast);
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram:" + e.getMessage());
- }
-
- return dmlPgm;
- }
-
- private DMLProgram createDMLProgram(DmlprogramContext ast) {
-
- DMLProgram dmlPgm = new DMLProgram();
-
- // First add all the functions
- for(FunctionStatementContext fn : ast.functionBlocks) {
- FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
- functionStmtBlk.addStatement(fn.info.stmt);
- try {
- // TODO: currently the logic of nested namespace is not clear.
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
- } catch (LanguageException e) {
- LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
- return null;
- }
- }
-
- // Then add all the statements
- for(StatementContext stmtCtx : ast.blocks) {
- com.ibm.bi.dml.parser.Statement current = stmtCtx.info.stmt;
- if(current == null) {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
- return null;
- }
-
- if(current instanceof ImportStatement) {
- // Handle import statements separately
- if(stmtCtx.info.namespaces != null) {
- // Add the DMLProgram entries into current program
- for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
- dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-
-// // Don't add DMLProgram into the current program, just add function statements
- // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
- // Add function statements to current dml program
-// DMLProgram importedPgm = entry.getValue();
-// try {
-// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
-// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
-// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
-// System.out.println("Adding function => " + entry.getKey() + "::" + functionName);
-// TODO:33
-// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
-// }
-// else {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
-// return null;
-// }
-// }
-// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
-// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
-// }
-// } catch (LanguageException e) {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement: " + e.getMessage());
-// return null;
-// }
- }
- }
- else {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
- return null;
- }
- }
-
- // Now wrap statement into individual statement block
- // merge statement will take care of merging these blocks
- dmlPgm.addStatementBlock(getStatementBlock(current));
- }
-
- dmlPgm.mergeStatementBlocks();
- return dmlPgm;
- }
-
- public static String readDMLScript( String script )
- throws IOException, LanguageException
- {
- String dmlScriptStr = null;
-
- //read DML script from file
- if(script == null)
- throw new LanguageException("DML script path was not specified!");
-
- StringBuilder sb = new StringBuilder();
- BufferedReader in = null;
- try
- {
- //read from hdfs or gpfs file system
- if( script.startsWith("hdfs:")
- || script.startsWith("gpfs:") )
- {
- if( !LocalFileUtils.validateExternalFilename(script, true) )
- throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path scriptPath = new Path(script);
- in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
- }
- // from local file system
- else
- {
- if( !LocalFileUtils.validateExternalFilename(script, false) )
- throw new LanguageException("Invalid (non-trustworthy) local filename.");
- in = new BufferedReader(new FileReader(script));
- }
-
- //core script reading
- String tmp = null;
- while ((tmp = in.readLine()) != null)
- {
- sb.append( tmp );
- sb.append( "\n" );
- }
- }
- catch (IOException ex)
- {
- LOG.error("Failed to read the script from the file system", ex);
- throw ex;
- }
- finally
- {
- if( in != null )
- in.close();
- }
-
- dmlScriptStr = sb.toString();
-
- return dmlScriptStr;
- }
-}
[73/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/FunctionStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/FunctionStatement.java b/src/main/java/org/apache/sysml/parser/FunctionStatement.java
index 98d58d0..ddad0cc 100644
--- a/src/main/java/org/apache/sysml/parser/FunctionStatement.java
+++ b/src/main/java/org/apache/sysml/parser/FunctionStatement.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.lops.Lop;
public class FunctionStatement extends Statement
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/FunctionStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/FunctionStatementBlock.java b/src/main/java/org/apache/sysml/parser/FunctionStatementBlock.java
index faf9ac5..583501f 100644
--- a/src/main/java/org/apache/sysml/parser/FunctionStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/FunctionStatementBlock.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.FunctionOp.FunctionType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class FunctionStatementBlock extends StatementBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/Identifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Identifier.java b/src/main/java/org/apache/sysml/parser/Identifier.java
index 522b513..774d7f0 100644
--- a/src/main/java/org/apache/sysml/parser/Identifier.java
+++ b/src/main/java/org/apache/sysml/parser/Identifier.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
public abstract class Identifier extends Expression
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/IfStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatement.java b/src/main/java/org/apache/sysml/parser/IfStatement.java
index 620d9de..95d2dd4 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatement.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
index 4de36e5..1fc904c 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
public class IfStatementBlock extends StatementBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ImportStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ImportStatement.java b/src/main/java/org/apache/sysml/parser/ImportStatement.java
index 87c9a4d..4f7414b 100644
--- a/src/main/java/org/apache/sysml/parser/ImportStatement.java
+++ b/src/main/java/org/apache/sysml/parser/ImportStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
index a23c663..c8a5446 100644
--- a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/IntIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IntIdentifier.java b/src/main/java/org/apache/sysml/parser/IntIdentifier.java
index cb5446c..f5bb820 100644
--- a/src/main/java/org/apache/sysml/parser/IntIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/IntIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/IterablePredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IterablePredicate.java b/src/main/java/org/apache/sysml/parser/IterablePredicate.java
index 0fab6db..b84fda0 100644
--- a/src/main/java/org/apache/sysml/parser/IterablePredicate.java
+++ b/src/main/java/org/apache/sysml/parser/IterablePredicate.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.lops.Lop;
public class IterablePredicate extends Expression
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/LanguageException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/LanguageException.java b/src/main/java/org/apache/sysml/parser/LanguageException.java
index 6fd636a..13002ae 100644
--- a/src/main/java/org/apache/sysml/parser/LanguageException.java
+++ b/src/main/java/org/apache/sysml/parser/LanguageException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
/**
* <p>Exception occurring at the Language level.</p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java b/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
index 735dc92..338ced2 100644
--- a/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
+++ b/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
public abstract class LiveVariableAnalysis
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/MultiAssignmentStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/MultiAssignmentStatement.java b/src/main/java/org/apache/sysml/parser/MultiAssignmentStatement.java
index 66e48b1..7139432 100644
--- a/src/main/java/org/apache/sysml/parser/MultiAssignmentStatement.java
+++ b/src/main/java/org/apache/sysml/parser/MultiAssignmentStatement.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DMLBreakpointManager;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/OutputStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/OutputStatement.java b/src/main/java/org/apache/sysml/parser/OutputStatement.java
index 3b9d6d6..8a2aff7 100644
--- a/src/main/java/org/apache/sysml/parser/OutputStatement.java
+++ b/src/main/java/org/apache/sysml/parser/OutputStatement.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-import com.ibm.bi.dml.parser.Expression.DataOp;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DMLBreakpointManager;
+import org.apache.sysml.parser.Expression.DataOp;
public class OutputStatement extends Statement
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ParForStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParForStatement.java b/src/main/java/org/apache/sysml/parser/ParForStatement.java
index c8b1bf6..0e2feee 100644
--- a/src/main/java/org/apache/sysml/parser/ParForStatement.java
+++ b/src/main/java/org/apache/sysml/parser/ParForStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
/**
* This ParForStatement is essentially identical to a ForStatement, except an extended
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
index e2ae2d7..58bed91 100644
--- a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,21 +29,21 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.parser.Expression.BinaryOp;
-import com.ibm.bi.dml.parser.Expression.BuiltinFunctionOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.parser.Expression.BinaryOp;
+import org.apache.sysml.parser.Expression.BuiltinFunctionOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.PrintStatement.PRINTTYPE;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/**
*
@@ -147,7 +147,7 @@ public class ParForStatementBlock extends ForStatementBlock
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.parser.ParForStatementBlock")
+ Logger.getLogger("org.apache.sysml.parser.ParForStatementBlock")
.setLevel((Level) Level.TRACE);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ParameterExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterExpression.java b/src/main/java/org/apache/sysml/parser/ParameterExpression.java
index 2745384..75b746a 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterExpression.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
public class ParameterExpression
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index 979cd22..13700bf 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParseException.java b/src/main/java/org/apache/sysml/parser/ParseException.java
index 86ff297..db41bd0 100644
--- a/src/main/java/org/apache/sysml/parser/ParseException.java
+++ b/src/main/java/org/apache/sysml/parser/ParseException.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
/**
* This exception is thrown when parse errors are encountered.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/PathStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/PathStatement.java b/src/main/java/org/apache/sysml/parser/PathStatement.java
index 9a9280e..d820345 100644
--- a/src/main/java/org/apache/sysml/parser/PathStatement.java
+++ b/src/main/java/org/apache/sysml/parser/PathStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/PrintStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/PrintStatement.java b/src/main/java/org/apache/sysml/parser/PrintStatement.java
index aac58ea..b74128d 100644
--- a/src/main/java/org/apache/sysml/parser/PrintStatement.java
+++ b/src/main/java/org/apache/sysml/parser/PrintStatement.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DMLBreakpointManager;
public class PrintStatement extends Statement
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/RelationalExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/RelationalExpression.java b/src/main/java/org/apache/sysml/parser/RelationalExpression.java
index 06f76f2..d3d6107 100644
--- a/src/main/java/org/apache/sysml/parser/RelationalExpression.java
+++ b/src/main/java/org/apache/sysml/parser/RelationalExpression.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
public class RelationalExpression extends Expression
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/Statement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Statement.java b/src/main/java/org/apache/sysml/parser/Statement.java
index bf76ce0..00ac236 100644
--- a/src/main/java/org/apache/sysml/parser/Statement.java
+++ b/src/main/java/org/apache/sysml/parser/Statement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index 437d859..b9d129e 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,19 +24,19 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.FormatType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.FormatType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.parser.PrintStatement.PRINTTYPE;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
public class StatementBlock extends LiveVariableAnalysis
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/StringIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StringIdentifier.java b/src/main/java/org/apache/sysml/parser/StringIdentifier.java
index 260cfe6..1add275 100644
--- a/src/main/java/org/apache/sysml/parser/StringIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/StringIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/Token.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Token.java b/src/main/java/org/apache/sysml/parser/Token.java
index a0b0bef..372ced9 100644
--- a/src/main/java/org/apache/sysml/parser/Token.java
+++ b/src/main/java/org/apache/sysml/parser/Token.java
@@ -1,6 +1,6 @@
/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
/**
* Describes the input token stream.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/VariableSet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/VariableSet.java b/src/main/java/org/apache/sysml/parser/VariableSet.java
index f99ca5f..675dfef 100644
--- a/src/main/java/org/apache/sysml/parser/VariableSet.java
+++ b/src/main/java/org/apache/sysml/parser/VariableSet.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/WhileStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/WhileStatement.java b/src/main/java/org/apache/sysml/parser/WhileStatement.java
index 740b31b..d01da06 100644
--- a/src/main/java/org/apache/sysml/parser/WhileStatement.java
+++ b/src/main/java/org/apache/sysml/parser/WhileStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
index 679fbee..c089001 100644
--- a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
public class WhileStatementBlock extends StatementBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
index 5cf6cdd..a967132 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -37,29 +37,29 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DmlprogramContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.antlr4.DmlParser.DmlprogramContext;
+import org.apache.sysml.parser.antlr4.DmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.StatementContext;
+import org.apache.sysml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
+import org.apache.sysml.runtime.util.LocalFileUtils;
/**
* This is the main entry point for the Antlr4 parser.
@@ -273,7 +273,7 @@ public class DMLParserWrapper extends AParserWrapper
// Then add all the statements
for(StatementContext stmtCtx : ast.blocks) {
- com.ibm.bi.dml.parser.Statement current = stmtCtx.info.stmt;
+ org.apache.sysml.parser.Statement current = stmtCtx.info.stmt;
if(current == null) {
LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
return null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4 b/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
index 4911088..7c52c47 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
+++ b/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
@@ -20,7 +20,7 @@ grammar Dml;
@header
{
// Commenting the package name and explicitly passing it in build.xml to maintain compatibility with maven plugin
- // package com.ibm.bi.dml.antlr4;
+ // package org.apache.sysml.antlr4;
}
// DML Program is a list of expression
@@ -91,7 +91,7 @@ dataIdentifier returns [ ExpressionInfo dataInfo ]
@init {
// This actions occurs regardless of how many alternatives in this rule
$dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new com.ibm.bi.dml.parser.DataIdentifier();
+ // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
} :
// ------------------------------------------
// IndexedIdentifier
@@ -105,7 +105,7 @@ expression returns [ ExpressionInfo info ]
@init {
// This actions occurs regardless of how many alternatives in this rule
$info = new ExpressionInfo();
- // $info.expr = new com.ibm.bi.dml.parser.BinaryExpression(com.ibm.bi.dml.parser.Expression.BinaryOp.INVALID);
+ // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
} :
// ------------------------------------------
// BinaryExpression
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
index 5fdf2d8..fa6489a 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.RecognitionException;
@@ -23,7 +23,7 @@ import org.antlr.v4.runtime.Recognizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
+import org.apache.sysml.api.DMLScript;
import java.util.Stack;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
index f19f618..521a0f8 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
import java.io.File;
import java.util.ArrayList;
@@ -28,92 +28,92 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
-import com.ibm.bi.dml.parser.ConditionalPredicate;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DoubleIdentifier;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.Expression.DataOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AddSubExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AtomicExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanAndExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanNotExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanOrExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BuiltinFunctionExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.CommandlineParamExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.CommandlinePositionExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstDoubleIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstFalseExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstIntIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstStringIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstTrueExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DataIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DataIdentifierContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DmlprogramContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ExternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ForStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallMultiAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IfStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IfdefAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ImportStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IndexedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.InternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IterablePredicateColonExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IterablePredicateSeqExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MatrixDataTypeCheckContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MatrixMulExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.Ml_typeContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ModIntDivExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MultDivExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParForStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.PathStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.PowerExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.RelationalExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.SimpleDataIdentifierExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StrictParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StrictParameterizedKeyValueStringContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.TypedArgNoAssignContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.UnaryExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ValueTypeContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.WhileStatementContext;
-import com.ibm.bi.dml.parser.AssignmentStatement;
-import com.ibm.bi.dml.parser.BinaryExpression;
-import com.ibm.bi.dml.parser.BooleanExpression;
-import com.ibm.bi.dml.parser.BooleanIdentifier;
-import com.ibm.bi.dml.parser.BuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ConstIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.FunctionCallIdentifier;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.IndexedIdentifier;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.IterablePredicate;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.MultiAssignmentStatement;
-import com.ibm.bi.dml.parser.OutputStatement;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParameterExpression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.PathStatement;
-import com.ibm.bi.dml.parser.PrintStatement;
-import com.ibm.bi.dml.parser.RelationalExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.WhileStatement;
+import org.apache.sysml.parser.ConditionalPredicate;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DoubleIdentifier;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.Expression.DataOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.antlr4.DmlParser.AddSubExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.AssignmentStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.AtomicExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.BooleanAndExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.BooleanNotExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.BooleanOrExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.BuiltinFunctionExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.CommandlineParamExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.CommandlinePositionExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ConstDoubleIdExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ConstFalseExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ConstIntIdExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ConstStringIdExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ConstTrueExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.DataIdExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.DataIdentifierContext;
+import org.apache.sysml.parser.antlr4.DmlParser.DmlprogramContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ExternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ForStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallMultiAssignmentStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.IfStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.IfdefAssignmentStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ImportStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.IndexedExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.InternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.IterablePredicateColonExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.IterablePredicateSeqExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.MatrixDataTypeCheckContext;
+import org.apache.sysml.parser.antlr4.DmlParser.MatrixMulExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.Ml_typeContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ModIntDivExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.MultDivExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ParForStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.PathStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.PowerExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.RelationalExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.SimpleDataIdentifierExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.StatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.StrictParameterizedExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.StrictParameterizedKeyValueStringContext;
+import org.apache.sysml.parser.antlr4.DmlParser.TypedArgNoAssignContext;
+import org.apache.sysml.parser.antlr4.DmlParser.UnaryExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ValueTypeContext;
+import org.apache.sysml.parser.antlr4.DmlParser.WhileStatementContext;
+import org.apache.sysml.parser.AssignmentStatement;
+import org.apache.sysml.parser.BinaryExpression;
+import org.apache.sysml.parser.BooleanExpression;
+import org.apache.sysml.parser.BooleanIdentifier;
+import org.apache.sysml.parser.BuiltinFunctionExpression;
+import org.apache.sysml.parser.ConstIdentifier;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.FunctionCallIdentifier;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.IndexedIdentifier;
+import org.apache.sysml.parser.IntIdentifier;
+import org.apache.sysml.parser.IterablePredicate;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.MultiAssignmentStatement;
+import org.apache.sysml.parser.OutputStatement;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParameterExpression;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.PathStatement;
+import org.apache.sysml.parser.PrintStatement;
+import org.apache.sysml.parser.RelationalExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.StringIdentifier;
+import org.apache.sysml.parser.WhileStatement;
public class DmlSyntacticValidator implements DmlListener
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
index 1d0f675..33ad63d 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
import java.util.ArrayList;
import java.util.List;
import org.antlr.v4.runtime.Token;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
public class DmlSyntacticValidatorHelper {
@@ -47,16 +47,16 @@ public class DmlSyntacticValidatorHelper {
return _errorListener.peekFileName();
}
-// public static void setInfoForArithmeticOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
+// public static void setInfoForArithmeticOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
// try {
// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// com.ibm.bi.dml.parser.Expression.BinaryOp bop = com.ibm.bi.dml.parser.Expression.getBinaryOp(opStr);
-// current = new com.ibm.bi.dml.parser.BinaryExpression(bop);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// org.apache.sysml.parser.Expression.BinaryOp bop = org.apache.sysml.parser.Expression.getBinaryOp(opStr);
+// current = new org.apache.sysml.parser.BinaryExpression(bop);
+// ((org.apache.sysml.parser.BinaryExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BinaryExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
// }
// catch(Exception e) {
// System.out.println("In setInfoForArithmeticOp>>");
@@ -64,14 +64,14 @@ public class DmlSyntacticValidatorHelper {
// }
// }
-// public static void setInfoForBooleanOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// com.ibm.bi.dml.parser.Expression.BooleanOp bop = com.ibm.bi.dml.parser.Expression.getBooleanOp(opStr);
-// current = new com.ibm.bi.dml.parser.BooleanExpression(bop);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// public static void setInfoForBooleanOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
+// org.apache.sysml.parser.Expression.BooleanOp bop = org.apache.sysml.parser.Expression.getBooleanOp(opStr);
+// current = new org.apache.sysml.parser.BooleanExpression(bop);
+// ((org.apache.sysml.parser.BooleanExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BooleanExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
// }
public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
@@ -87,14 +87,14 @@ public class DmlSyntacticValidatorHelper {
return true;
}
- public ArrayList<com.ibm.bi.dml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<com.ibm.bi.dml.parser.ParameterExpression> retVal = new ArrayList<com.ibm.bi.dml.parser.ParameterExpression>();
+ public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
+ ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
for(ParameterizedExpressionContext ctx : paramExprs) {
String paramName = null;
if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
paramName = ctx.paramName.getText();
}
- com.ibm.bi.dml.parser.ParameterExpression myArg = new com.ibm.bi.dml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
+ org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
retVal.add(myArg);
}
return retVal;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
index d7ea7eb..c08bada 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
public class ExpressionInfo {
- public com.ibm.bi.dml.parser.Expression expr = null;
+ public org.apache.sysml.parser.Expression expr = null;
// For parfor and for
- public com.ibm.bi.dml.parser.Expression from = null;
- public com.ibm.bi.dml.parser.Expression to = null;
- public com.ibm.bi.dml.parser.Expression increment = null;
+ public org.apache.sysml.parser.Expression from = null;
+ public org.apache.sysml.parser.Expression to = null;
+ public org.apache.sysml.parser.Expression increment = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java b/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
index e32e21a..0037862 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.parser.antlr4;
+package org.apache.sysml.parser.antlr4;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLProgram;
public class StatementInfo {
- public com.ibm.bi.dml.parser.Statement stmt = null;
+ public org.apache.sysml.parser.Statement stmt = null;
// Valid only for import statements
public HashMap<String,DMLProgram> namespaces = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
index 09e8c27..6e4a464 100644
--- a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
+++ b/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
/**
* This class exists solely to prevent compiler warnings.
@@ -8,6 +8,6 @@ package com.ibm.bi.dml.parser.python;
* (R-like and Python-like dialects), and Antlr-generated code assumes that
* these classes are present in the parser's namespace.
*/
-class ExpressionInfo extends com.ibm.bi.dml.parser.antlr4.ExpressionInfo {
+class ExpressionInfo extends org.apache.sysml.parser.antlr4.ExpressionInfo {
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
index 69fb746..f57c3ac 100644
--- a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
@@ -35,28 +35,28 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.antlr4.DMLParserWrapper;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PmlprogramContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StatementContext;
-import com.ibm.bi.dml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.antlr4.DMLParserWrapper;
+import org.apache.sysml.parser.python.PydmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.PmlprogramContext;
+import org.apache.sysml.parser.python.PydmlParser.StatementContext;
+import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
/**
* Logic of this wrapper is similar to DMLParserWrapper.
@@ -73,7 +73,7 @@ public class PyDMLParserWrapper extends AParserWrapper
* @param current a statement
* @return corresponding statement block
*/
- public static StatementBlock getStatementBlock(com.ibm.bi.dml.parser.Statement current) {
+ public static StatementBlock getStatementBlock(org.apache.sysml.parser.Statement current) {
StatementBlock blk = null;
if(current instanceof ParForStatement) {
blk = new ParForStatementBlock();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/Pydml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/Pydml.g4 b/src/main/java/org/apache/sysml/parser/python/Pydml.g4
index 821ca29..b03e3ec 100644
--- a/src/main/java/org/apache/sysml/parser/python/Pydml.g4
+++ b/src/main/java/org/apache/sysml/parser/python/Pydml.g4
@@ -19,9 +19,9 @@ grammar Pydml;
@header
{
- // package com.ibm.bi.dml.python;
- //import com.ibm.bi.dml.parser.antlr4.StatementInfo;
- //import com.ibm.bi.dml.parser.antlr4.ExpressionInfo;
+ // package org.apache.sysml.python;
+ //import org.apache.sysml.parser.antlr4.StatementInfo;
+ //import org.apache.sysml.parser.antlr4.ExpressionInfo;
}
// This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html
@@ -200,7 +200,7 @@ dataIdentifier returns [ ExpressionInfo dataInfo ]
@init {
// This actions occurs regardless of how many alternatives in this rule
$dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new com.ibm.bi.dml.parser.DataIdentifier();
+ // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
} :
// ------------------------------------------
// IndexedIdentifier
@@ -214,7 +214,7 @@ expression returns [ ExpressionInfo info ]
@init {
// This actions occurs regardless of how many alternatives in this rule
$info = new ExpressionInfo();
- // $info.expr = new com.ibm.bi.dml.parser.BinaryExpression(com.ibm.bi.dml.parser.Expression.BinaryOp.INVALID);
+ // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
} :
// ------------------------------------------
// BinaryExpression
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
index 8a29038..c3e88d0 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
import java.util.Stack;
@@ -25,7 +25,7 @@ import org.antlr.v4.runtime.Recognizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
+import org.apache.sysml.api.DMLScript;
public class PydmlSyntacticErrorListener {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
index 88b6138..1fb5e8b 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
import java.io.File;
import java.util.ArrayList;
@@ -29,94 +29,94 @@ import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringUtils;
-import com.ibm.bi.dml.parser.AssignmentStatement;
-import com.ibm.bi.dml.parser.BinaryExpression;
-import com.ibm.bi.dml.parser.BooleanExpression;
-import com.ibm.bi.dml.parser.BooleanIdentifier;
-import com.ibm.bi.dml.parser.BuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ConditionalPredicate;
-import com.ibm.bi.dml.parser.ConstIdentifier;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DoubleIdentifier;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.Expression.DataOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.FunctionCallIdentifier;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.IndexedIdentifier;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.IterablePredicate;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.MultiAssignmentStatement;
-import com.ibm.bi.dml.parser.OutputStatement;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParameterExpression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.PathStatement;
-import com.ibm.bi.dml.parser.PrintStatement;
-import com.ibm.bi.dml.parser.RelationalExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.WhileStatement;
-//import com.ibm.bi.dml.parser.antlr4.ExpressionInfo;
-//import com.ibm.bi.dml.parser.antlr4.StatementInfo;
-import com.ibm.bi.dml.parser.python.PydmlParser.AddSubExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.AssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.AtomicExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanAndExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanNotExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanOrExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BuiltinFunctionExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.CommandlineParamExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.CommandlinePositionExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstDoubleIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstFalseExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstIntIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstStringIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstTrueExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.DataIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.DataIdentifierContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ExternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ForStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallMultiAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IfStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IfdefAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IgnoreNewLineContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ImportStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IndexedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.InternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IterablePredicateColonExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IterablePredicateSeqExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.MatrixDataTypeCheckContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.Ml_typeContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ModIntDivExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.MultDivExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParForStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PathStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PmlprogramContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PowerExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.RelationalExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.SimpleDataIdentifierExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StrictParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StrictParameterizedKeyValueStringContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.TypedArgNoAssignContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.UnaryExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ValueDataTypeCheckContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.WhileStatementContext;
+import org.apache.sysml.parser.AssignmentStatement;
+import org.apache.sysml.parser.BinaryExpression;
+import org.apache.sysml.parser.BooleanExpression;
+import org.apache.sysml.parser.BooleanIdentifier;
+import org.apache.sysml.parser.BuiltinFunctionExpression;
+import org.apache.sysml.parser.ConditionalPredicate;
+import org.apache.sysml.parser.ConstIdentifier;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DoubleIdentifier;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.Expression.DataOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.FunctionCallIdentifier;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.IndexedIdentifier;
+import org.apache.sysml.parser.IntIdentifier;
+import org.apache.sysml.parser.IterablePredicate;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.MultiAssignmentStatement;
+import org.apache.sysml.parser.OutputStatement;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParameterExpression;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.PathStatement;
+import org.apache.sysml.parser.PrintStatement;
+import org.apache.sysml.parser.RelationalExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.StringIdentifier;
+import org.apache.sysml.parser.WhileStatement;
+//import org.apache.sysml.parser.antlr4.ExpressionInfo;
+//import org.apache.sysml.parser.antlr4.StatementInfo;
+import org.apache.sysml.parser.python.PydmlParser.AddSubExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.AssignmentStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.AtomicExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.BooleanAndExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.BooleanNotExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.BooleanOrExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.BuiltinFunctionExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.CommandlineParamExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.CommandlinePositionExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ConstDoubleIdExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ConstFalseExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ConstIntIdExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ConstStringIdExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ConstTrueExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.DataIdExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.DataIdentifierContext;
+import org.apache.sysml.parser.python.PydmlParser.ExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ExternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ForStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.FunctionCallMultiAssignmentStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.IfStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.IfdefAssignmentStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.IgnoreNewLineContext;
+import org.apache.sysml.parser.python.PydmlParser.ImportStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.IndexedExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.InternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.IterablePredicateColonExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.IterablePredicateSeqExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.MatrixDataTypeCheckContext;
+import org.apache.sysml.parser.python.PydmlParser.Ml_typeContext;
+import org.apache.sysml.parser.python.PydmlParser.ModIntDivExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.MultDivExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ParForStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.PathStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.PmlprogramContext;
+import org.apache.sysml.parser.python.PydmlParser.PowerExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.RelationalExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.SimpleDataIdentifierExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.StatementContext;
+import org.apache.sysml.parser.python.PydmlParser.StrictParameterizedExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.StrictParameterizedKeyValueStringContext;
+import org.apache.sysml.parser.python.PydmlParser.TypedArgNoAssignContext;
+import org.apache.sysml.parser.python.PydmlParser.UnaryExpressionContext;
+import org.apache.sysml.parser.python.PydmlParser.ValueDataTypeCheckContext;
+import org.apache.sysml.parser.python.PydmlParser.WhileStatementContext;
public class PydmlSyntacticValidator implements PydmlListener
{
@@ -1085,7 +1085,7 @@ public class PydmlSyntacticValidator implements PydmlListener
paramExpression.get(0).setName("rows");
paramExpression.get(1).setName("cols");
paramExpression.get(2).setName("sparsity");
- paramExpression.add(new com.ibm.bi.dml.parser.ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col)));
+ paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col)));
functionName = "rand";
namespace = DMLProgram.DEFAULT_NAMESPACE;
}
@@ -1099,7 +1099,7 @@ public class PydmlSyntacticValidator implements PydmlListener
paramExpression.get(2).setName("sparsity");
paramExpression.get(3).setName("min");
paramExpression.get(4).setName("max");
- paramExpression.add(new com.ibm.bi.dml.parser.ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col)));
+ paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col)));
functionName = "rand";
namespace = DMLProgram.DEFAULT_NAMESPACE;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
index 2805d9c..68bfa93 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
import java.util.ArrayList;
import java.util.List;
import org.antlr.v4.runtime.Token;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
public class PydmlSyntacticValidatorHelper {
@@ -68,16 +68,16 @@ public class PydmlSyntacticValidatorHelper {
return retVal;
}
-// public static void setInfoForArithmeticOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
+// public static void setInfoForArithmeticOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
// try {
// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// com.ibm.bi.dml.parser.Expression.BinaryOp bop = com.ibm.bi.dml.parser.Expression.getBinaryOp(opStr);
-// current = new com.ibm.bi.dml.parser.BinaryExpression(bop);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// org.apache.sysml.parser.Expression.BinaryOp bop = org.apache.sysml.parser.Expression.getBinaryOp(opStr);
+// current = new org.apache.sysml.parser.BinaryExpression(bop);
+// ((org.apache.sysml.parser.BinaryExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BinaryExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
// }
// catch(Exception e) {
// System.out.println("In setInfoForArithmeticOp>>");
@@ -85,14 +85,14 @@ public class PydmlSyntacticValidatorHelper {
// }
// }
-// public static void setInfoForBooleanOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// com.ibm.bi.dml.parser.Expression.BooleanOp bop = com.ibm.bi.dml.parser.Expression.getBooleanOp(opStr);
-// current = new com.ibm.bi.dml.parser.BooleanExpression(bop);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// public static void setInfoForBooleanOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
+// org.apache.sysml.parser.Expression.BooleanOp bop = org.apache.sysml.parser.Expression.getBooleanOp(opStr);
+// current = new org.apache.sysml.parser.BooleanExpression(bop);
+// ((org.apache.sysml.parser.BooleanExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BooleanExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
// }
public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
@@ -108,14 +108,14 @@ public class PydmlSyntacticValidatorHelper {
return true;
}
- public ArrayList<com.ibm.bi.dml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<com.ibm.bi.dml.parser.ParameterExpression> retVal = new ArrayList<com.ibm.bi.dml.parser.ParameterExpression>();
+ public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
+ ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
for(ParameterizedExpressionContext ctx : paramExprs) {
String paramName = null;
if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
paramName = ctx.paramName.getText();
}
- com.ibm.bi.dml.parser.ParameterExpression myArg = new com.ibm.bi.dml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
+ org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
retVal.add(myArg);
}
return retVal;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/StatementInfo.java b/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
index 82417a5..d0f07c2 100644
--- a/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
+++ b/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.parser.python;
+package org.apache.sysml.parser.python;
/**
* This class exists solely to prevent compiler warnings.
@@ -8,6 +8,6 @@ package com.ibm.bi.dml.parser.python;
* (R-like and Python-like dialects), and Antlr-generated code assumes that
* these classes are present in the parser's namespace.
*/
-class StatementInfo extends com.ibm.bi.dml.parser.antlr4.StatementInfo {
+class StatementInfo extends org.apache.sysml.parser.antlr4.StatementInfo {
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/DMLRuntimeException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/DMLRuntimeException.java b/src/main/java/org/apache/sysml/runtime/DMLRuntimeException.java
index 2bee04c..3a5ae49 100644
--- a/src/main/java/org/apache/sysml/runtime/DMLRuntimeException.java
+++ b/src/main/java/org/apache/sysml/runtime/DMLRuntimeException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime;
+package org.apache.sysml.runtime;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
/**
* This exception should be thrown to flag runtime errors -- DML equivalent to java.lang.RuntimeException.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/DMLScriptException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/DMLScriptException.java b/src/main/java/org/apache/sysml/runtime/DMLScriptException.java
index 8caa3e4..30c29f0 100644
--- a/src/main/java/org/apache/sysml/runtime/DMLScriptException.java
+++ b/src/main/java/org/apache/sysml/runtime/DMLScriptException.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime;
+package org.apache.sysml.runtime;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/DMLUnsupportedOperationException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/DMLUnsupportedOperationException.java b/src/main/java/org/apache/sysml/runtime/DMLUnsupportedOperationException.java
index c968a03..7096a53 100644
--- a/src/main/java/org/apache/sysml/runtime/DMLUnsupportedOperationException.java
+++ b/src/main/java/org/apache/sysml/runtime/DMLUnsupportedOperationException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime;
+package org.apache.sysml.runtime;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
public class DMLUnsupportedOperationException extends DMLException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index b48472c..07ae9b6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import java.util.Collection;
@@ -24,44 +24,44 @@ import java.util.HashSet;
import java.util.StringTokenizer;
import java.util.TreeMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.FunctionParameter.FunctionParameterType;
-import com.ibm.bi.dml.udf.BinaryObject;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.ReBlock;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Scalar;
+import org.apache.sysml.udf.FunctionParameter.FunctionParameterType;
+import org.apache.sysml.udf.BinaryObject;
+import org.apache.sysml.udf.Scalar.ScalarValueType;
public class ExternalFunctionProgramBlock extends FunctionProgramBlock
{
@@ -916,7 +916,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @return
*/
- protected com.ibm.bi.dml.udf.Matrix.ValueType getMatrixValueType(String string) {
+ protected org.apache.sysml.udf.Matrix.ValueType getMatrixValueType(String string) {
if (string.equals("Double"))
return Matrix.ValueType.Double;
[24/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java b/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
deleted file mode 100644
index e5b63f5..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
+++ /dev/null
@@ -1,4574 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.compile;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.AppendM;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Data.OperationTypes;
-import com.ibm.bi.dml.lops.FunctionCallCP;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.Lop.Type;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.OutputParameters;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.SPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat;
-
-
-
-/**
- *
- * Class to maintain a DAG and compile it into jobs
- * @param <N>
- */
-public class Dag<N extends Lop>
-{
-
- private static final Log LOG = LogFactory.getLog(Dag.class.getName());
-
- private static final int CHILD_BREAKS_ALIGNMENT = 2;
- private static final int CHILD_DOES_NOT_BREAK_ALIGNMENT = 1;
- private static final int MRCHILD_NOT_FOUND = 0;
- private static final int MR_CHILD_FOUND_BREAKS_ALIGNMENT = 4;
- private static final int MR_CHILD_FOUND_DOES_NOT_BREAK_ALIGNMENT = 5;
-
- private static IDSequence job_id = null;
- private static IDSequence var_index = null;
-
- private int total_reducers = -1;
- private String scratch = "";
- private String scratchFilePath = "";
-
- private double gmrMapperFootprint = 0;
-
- static {
- job_id = new IDSequence();
- var_index = new IDSequence();
- }
-
- // hash set for all nodes in dag
- private ArrayList<N> nodes = null;
-
- /*
- * Hashmap to translates the nodes in the DAG to a sequence of numbers
- * key: Lop ID
- * value: Sequence Number (0 ... |DAG|)
- *
- * This map is primarily used in performing DFS on the DAG, and subsequently in performing ancestor-descendant checks.
- */
- private HashMap<Long, Integer> IDMap = null;
-
-
- private class NodeOutput {
- String fileName;
- String varName;
- OutputInfo outInfo;
- ArrayList<Instruction> preInstructions; // instructions added before a MR instruction
- ArrayList<Instruction> postInstructions; // instructions added after a MR instruction
- ArrayList<Instruction> lastInstructions;
-
- NodeOutput() {
- fileName = null;
- varName = null;
- outInfo = null;
- preInstructions = new ArrayList<Instruction>();
- postInstructions = new ArrayList<Instruction>();
- lastInstructions = new ArrayList<Instruction>();
- }
-
- public String getFileName() {
- return fileName;
- }
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
- public String getVarName() {
- return varName;
- }
- public void setVarName(String varName) {
- this.varName = varName;
- }
- public OutputInfo getOutInfo() {
- return outInfo;
- }
- public void setOutInfo(OutputInfo outInfo) {
- this.outInfo = outInfo;
- }
- public ArrayList<Instruction> getPreInstructions() {
- return preInstructions;
- }
- public void addPreInstruction(Instruction inst) {
- preInstructions.add(inst);
- }
- public ArrayList<Instruction> getPostInstructions() {
- return postInstructions;
- }
- public void addPostInstruction(Instruction inst) {
- postInstructions.add(inst);
- }
- public ArrayList<Instruction> getLastInstructions() {
- return lastInstructions;
- }
- public void addLastInstruction(Instruction inst) {
- lastInstructions.add(inst);
- }
-
- }
-
- private String getFilePath() {
- if ( scratchFilePath.equalsIgnoreCase("") ) {
- scratchFilePath = scratch + Lop.FILE_SEPARATOR
- + Lop.PROCESS_PREFIX + DMLScript.getUUID()
- + Lop.FILE_SEPARATOR + Lop.FILE_SEPARATOR
- + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR;
- }
- return scratchFilePath;
- }
-
- /**
- * Constructor
- */
- public Dag()
- {
- //allocate internal data structures
- nodes = new ArrayList<N>();
- IDMap = new HashMap<Long, Integer>();
-
- // get number of reducers from dml config
- DMLConfig conf = ConfigurationManager.getConfig();
- total_reducers = conf.getIntValue(DMLConfig.NUM_REDUCERS);
- }
-
- /**
- *
- * @param config
- * @return
- * @throws LopsException
- * @throws IOException
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public ArrayList<Instruction> getJobs(DMLConfig config)
- throws LopsException, IOException, DMLRuntimeException, DMLUnsupportedOperationException
- {
- return getJobs(null, config);
- }
-
- /**
- * Method to compile a dag generically
- *
- * @param config
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
-
- public ArrayList<Instruction> getJobs(StatementBlock sb, DMLConfig config)
- throws LopsException, IOException, DMLRuntimeException,
- DMLUnsupportedOperationException {
-
- if (config != null)
- {
- total_reducers = config.getIntValue(DMLConfig.NUM_REDUCERS);
- scratch = config.getTextValue(DMLConfig.SCRATCH_SPACE) + "/";
- }
-
- // hold all nodes in a vector (needed for ordering)
- ArrayList<N> node_v = new ArrayList<N>();
- node_v.addAll(nodes);
-
- /*
- * Sort the nodes by topological order.
- *
- * 1) All nodes with level i appear prior to the nodes in level i+1.
- * 2) All nodes within a level are ordered by their ID i.e., in the order
- * they are created
- */
- doTopologicalSort_strict_order(node_v);
-
- // do greedy grouping of operations
- ArrayList<Instruction> inst = doGreedyGrouping(sb, node_v);
-
- return inst;
-
- }
-
- /**
- * Method to remove transient reads that do not have a transient write
- *
- * @param nodeV
- * @param inst
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("unused")
- private void deleteUnwantedTransientReadVariables(ArrayList<N> nodeV,
- ArrayList<Instruction> inst) throws DMLRuntimeException,
- DMLUnsupportedOperationException {
- HashMap<String, N> labelNodeMapping = new HashMap<String, N>();
-
- LOG.trace("In delete unwanted variables");
-
- // first capture all transient read variables
- for (int i = 0; i < nodeV.size(); i++) {
- N node = nodeV.get(i);
-
- if (node.getExecLocation() == ExecLocation.Data
- && ((Data) node).isTransient()
- && ((Data) node).getOperationType() == OperationTypes.READ) {
- labelNodeMapping.put(node.getOutputParameters().getLabel(),
- node);
- }
- }
-
- // generate delete instructions for all transient read variables without
- // a transient write
- // first capture all transient write variables
- for (int i = 0; i < nodeV.size(); i++) {
- N node = nodeV.get(i);
-
- if (node.getExecLocation() == ExecLocation.Data
- && ((Data) node).isTransient()
- && ((Data) node).getOperationType() == OperationTypes.WRITE ) {
- if (node.getInputs().get(0).getExecLocation() == ExecLocation.Data) {
- // this transient write is NOT a result of actual computation, but is simple copy.
- // in such a case, preserve the input variable so that appropriate copy instruction (cpvar or GMR) is generated
- // therefore, remove the input label from labelNodeMapping
- labelNodeMapping.remove(node.getInputs().get(0).getOutputParameters().getLabel());
- }
- else {
- if(labelNodeMapping.containsKey(node.getOutputParameters().getLabel()))
- // corresponding transient read exists (i.e., the variable is updated)
- // in such a case, generate rmvar instruction so that OLD data gets deleted
- labelNodeMapping.remove(node.getOutputParameters().getLabel());
- }
- }
- }
-
- // generate RM instructions
- Instruction rm_inst = null;
- for( Entry<String, N> e : labelNodeMapping.entrySet() ) {
- String label = e.getKey();
- N node = e.getValue();
-
- if (((Data) node).getDataType() == DataType.SCALAR) {
- // if(DEBUG)
- // System.out.println("rmvar" + Lops.OPERAND_DELIMITOR + label);
- // inst.add(new VariableSimpleInstructions("rmvar" +
- // Lops.OPERAND_DELIMITOR + label));
-
- } else {
- rm_inst = VariableCPInstruction.prepareRemoveInstruction(label);
- rm_inst.setLocation(node);
-
- if( LOG.isTraceEnabled() )
- LOG.trace(rm_inst.toString());
- inst.add(rm_inst);
-
- }
- }
-
- }
-
- private void deleteUpdatedTransientReadVariables(StatementBlock sb, ArrayList<N> nodeV,
- ArrayList<Instruction> inst) throws DMLRuntimeException,
- DMLUnsupportedOperationException {
-
- if ( sb == null )
- return;
-
- LOG.trace("In delete updated variables");
-
- /*
- Set<String> in = sb.liveIn().getVariables().keySet();
- Set<String> out = sb.liveOut().getVariables().keySet();
- Set<String> updated = sb.variablesUpdated().getVariables().keySet();
-
- Set<String> intersection = in;
- intersection.retainAll(out);
- intersection.retainAll(updated);
-
- for (String var : intersection) {
- inst.add(VariableCPInstruction.prepareRemoveInstruction(var));
- }
- */
-
- // CANDIDATE list of variables which could have been updated in this statement block
- HashMap<String, N> labelNodeMapping = new HashMap<String, N>();
-
- // ACTUAL list of variables whose value is updated, AND the old value of the variable
- // is no longer accessible/used.
- HashSet<String> updatedLabels = new HashSet<String>();
- HashMap<String, N> updatedLabelsLineNum = new HashMap<String, N>();
-
- // first capture all transient read variables
- for (int i = 0; i < nodeV.size(); i++) {
- N node = nodeV.get(i);
-
- if (node.getExecLocation() == ExecLocation.Data
- && ((Data) node).isTransient()
- && ((Data) node).getOperationType() == OperationTypes.READ
- && ((Data) node).getDataType() == DataType.MATRIX) {
-
- // "node" is considered as updated ONLY IF the old value is not used any more
- // So, make sure that this READ node does not feed into any (transient/persistent) WRITE
- boolean hasWriteParent=false;
- for(Lop p : node.getOutputs()) {
- if(p.getExecLocation() == ExecLocation.Data) {
- // if the "p" is of type Data, then it has to be a WRITE
- hasWriteParent = true;
- break;
- }
- }
-
- if ( !hasWriteParent ) {
- // node has no parent of type WRITE, so this is a CANDIDATE variable
- // add it to labelNodeMapping so that it is considered in further processing
- labelNodeMapping.put(node.getOutputParameters().getLabel(), node);
- }
- }
- }
-
- // capture updated transient write variables
- for (int i = 0; i < nodeV.size(); i++) {
- N node = nodeV.get(i);
-
- if (node.getExecLocation() == ExecLocation.Data
- && ((Data) node).isTransient()
- && ((Data) node).getOperationType() == OperationTypes.WRITE
- && ((Data) node).getDataType() == DataType.MATRIX
- && labelNodeMapping.containsKey(node.getOutputParameters().getLabel()) // check to make sure corresponding (i.e., with the same label/name) transient read is present
- && !labelNodeMapping.containsValue(node.getInputs().get(0)) // check to avoid cases where transient read feeds into a transient write
- ) {
- updatedLabels.add(node.getOutputParameters().getLabel());
- updatedLabelsLineNum.put(node.getOutputParameters().getLabel(), node);
-
- }
- }
-
- // generate RM instructions
- Instruction rm_inst = null;
- for ( String label : updatedLabels )
- {
- rm_inst = VariableCPInstruction.prepareRemoveInstruction(label);
- rm_inst.setLocation(updatedLabelsLineNum.get(label));
-
-
- LOG.trace(rm_inst.toString());
- inst.add(rm_inst);
- }
-
- }
-
- private void generateRemoveInstructions(StatementBlock sb,
- ArrayList<Instruction> deleteInst)
- throws DMLUnsupportedOperationException, DMLRuntimeException {
-
- if ( sb == null )
- return;
-
- LOG.trace("In generateRemoveInstructions()");
-
-
- Instruction inst = null;
- // RULE 1: if in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- // (currently required for specific cases of external functions)
- for (String varName : sb.liveIn().getVariableNames()) {
- if (!sb.liveOut().containsVariable(varName)) {
- // DataType dt = in.getVariable(varName).getDataType();
- // if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- // continue; //skip rm instructions for non-matrix objects
-
- inst = VariableCPInstruction.prepareRemoveInstruction(varName);
- inst.setLocation(sb.getEndLine(), sb.getEndLine(), -1, -1);
-
- deleteInst.add(inst);
-
- if( LOG.isTraceEnabled() )
- LOG.trace(" Adding " + inst.toString());
- }
- }
-
- // RULE 2: if in KILL and not in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- // (currently required for specific cases of nested loops)
- // i.e., local variables which are created within the block, and used entirely within the block
- /*for (String varName : sb.getKill().getVariableNames()) {
- if ((!sb.liveIn().containsVariable(varName))
- && (!sb.liveOut().containsVariable(varName))) {
- // DataType dt =
- // sb.getKill().getVariable(varName).getDataType();
- // if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- // continue; //skip rm instructions for non-matrix objects
-
- inst = createCleanupInstruction(varName);
- deleteInst.add(inst);
-
- if (DMLScript.DEBUG)
- System.out.println("Adding instruction (r2) "
- + inst.toString());
- }
- }*/
- }
-
- /**
- * Method to add a node to the DAG.
- *
- * @param node
- * @return true if node was not already present, false if not.
- */
-
- public boolean addNode(N node) {
- if (nodes.contains(node))
- return false;
- else {
- nodes.add(node);
- return true;
- }
-
- }
-
- private ArrayList<ArrayList<N>> createNodeVectors(int size) {
- ArrayList<ArrayList<N>> arr = new ArrayList<ArrayList<N>>();
-
- // for each job type, we need to create a vector.
- // additionally, create another vector for execNodes
- for (int i = 0; i < size; i++) {
- arr.add(new ArrayList<N>());
- }
- return arr;
- }
-
- private void clearNodeVectors(ArrayList<ArrayList<N>> arr) {
- for (int i = 0; i < arr.size(); i++) {
- arr.get(i).clear();
- }
- }
-
- private boolean isCompatible(ArrayList<N> nodes, JobType jt, int from,
- int to) throws LopsException {
-
- int base = jt.getBase();
-
- for (int i = from; i < to; i++) {
- if ((nodes.get(i).getCompatibleJobs() & base) == 0) {
- if( LOG.isTraceEnabled() )
- LOG.trace("Not compatible "+ nodes.get(i).toString());
- return false;
- }
- }
- return true;
- }
-
- /**
- * Function that determines if the two input nodes can be executed together
- * in at least on job.
- *
- * @param node1
- * @param node2
- * @return
- */
- private boolean isCompatible(N node1, N node2) {
- return( (node1.getCompatibleJobs() & node2.getCompatibleJobs()) > 0);
- }
-
- /**
- * Function that checks if the given node executes in the job specified by jt.
- *
- * @param node
- * @param jt
- * @return
- */
- private boolean isCompatible(N node, JobType jt) {
- if ( jt == JobType.GMRCELL )
- jt = JobType.GMR;
- return ((node.getCompatibleJobs() & jt.getBase()) > 0);
- }
-
- /*
- * Add node, and its relevant children to job-specific node vectors.
- */
- private void addNodeByJobType(N node, ArrayList<ArrayList<N>> arr,
- ArrayList<N> execNodes, boolean eliminate) throws LopsException {
-
- if (!eliminate) {
- // Check if this lop defines a MR job.
- if ( node.definesMRJob() ) {
-
- // find the corresponding JobType
- JobType jt = JobType.findJobTypeFromLop(node);
-
- if ( jt == null ) {
- throw new LopsException(node.printErrorLocation() + "No matching JobType is found for a the lop type: " + node.getType() + " \n");
- }
-
- // Add "node" to corresponding job vector
-
- if ( jt == JobType.GMR ) {
- if ( node.hasNonBlockedInputs() ) {
- int gmrcell_index = JobType.GMRCELL.getId();
- arr.get(gmrcell_index).add(node);
- int from = arr.get(gmrcell_index).size();
- addChildren(node, arr.get(gmrcell_index), execNodes);
- int to = arr.get(gmrcell_index).size();
- if (!isCompatible(arr.get(gmrcell_index),JobType.GMR, from, to)) // check against GMR only, not against GMRCELL
- throw new LopsException(node.printErrorLocation() + "Error during compatibility check \n");
- }
- else {
- // if "node" (in this case, a group lop) has any inputs from RAND
- // then add it to RAND job. Otherwise, create a GMR job
- if (hasChildNode(node, arr.get(JobType.DATAGEN.getId()) )) {
- arr.get(JobType.DATAGEN.getId()).add(node);
- // we should NOT call 'addChildren' because appropriate
- // child nodes would have got added to RAND job already
- } else {
- int gmr_index = JobType.GMR.getId();
- arr.get(gmr_index).add(node);
- int from = arr.get(gmr_index).size();
- addChildren(node, arr.get(gmr_index), execNodes);
- int to = arr.get(gmr_index).size();
- if (!isCompatible(arr.get(gmr_index),JobType.GMR, from, to))
- throw new LopsException(node.printErrorLocation() + "Error during compatibility check \n");
- }
- }
- }
- else {
- int index = jt.getId();
- arr.get(index).add(node);
- int from = arr.get(index).size();
- addChildren(node, arr.get(index), execNodes);
- int to = arr.get(index).size();
- // check if all added nodes are compatible with current job
- if (!isCompatible(arr.get(index), jt, from, to)) {
- throw new LopsException(
- "Unexpected error in addNodeByType.");
- }
- }
- return;
- }
- }
-
- if ( eliminate ) {
- // Eliminated lops are directly added to GMR queue.
- // Note that eliminate flag is set only for 'group' lops
- if ( node.hasNonBlockedInputs() )
- arr.get(JobType.GMRCELL.getId()).add(node);
- else
- arr.get(JobType.GMR.getId()).add(node);
- return;
- }
-
- /*
- * If this lop does not define a job, check if it uses the output of any
- * specialized job. i.e., if this lop has a child node in any of the
- * job-specific vector, then add it to the vector. Note: This lop must
- * be added to ONLY ONE of the job-specific vectors.
- */
-
- int numAdded = 0;
- for ( JobType j : JobType.values() ) {
- if ( j.getId() > 0 && hasDirectChildNode(node, arr.get(j.getId()))) {
- if (isCompatible(node, j)) {
- arr.get(j.getId()).add(node);
- numAdded += 1;
- }
- }
- }
- if (numAdded > 1) {
- throw new LopsException("Unexpected error in addNodeByJobType(): A given lop can ONLY be added to a single job vector (numAdded = " + numAdded + ")." );
- }
- }
-
- /*
- * Remove the node from all job-specific node vectors. This method is
- * invoked from removeNodesForNextIteration().
- */
- private void removeNodeByJobType(N node, ArrayList<ArrayList<N>> arr) {
- for ( JobType jt : JobType.values())
- if ( jt.getId() > 0 )
- arr.get(jt.getId()).remove(node);
- }
-
- /**
- * As some jobs only write one output, all operations in the mapper need to
- * be redone and cannot be marked as finished.
- *
- * @param execNodes
- * @param jobNodes
- * @throws LopsException
- */
-
- private void handleSingleOutputJobs(ArrayList<N> execNodes,
- ArrayList<ArrayList<N>> jobNodes, ArrayList<N> finishedNodes)
- throws LopsException {
- /*
- * If the input of a MMCJ/MMRJ job (must have executed in a Mapper) is used
- * by multiple lops then we should mark it as not-finished.
- */
- ArrayList<N> nodesWithUnfinishedOutputs = new ArrayList<N>();
- int[] jobIndices = {JobType.MMCJ.getId()};
- Lop.Type[] lopTypes = { Lop.Type.MMCJ};
-
- // TODO: SortByValue should be treated similar to MMCJ, since it can
- // only sort one file now
-
- for ( int jobi=0; jobi < jobIndices.length; jobi++ ) {
- int jindex = jobIndices[jobi];
- if (!jobNodes.get(jindex).isEmpty()) {
- ArrayList<N> vec = jobNodes.get(jindex);
-
- // first find all nodes with more than one parent that is not
- // finished.
-
- for (int i = 0; i < vec.size(); i++) {
- N node = vec.get(i);
- if (node.getExecLocation() == ExecLocation.MapOrReduce
- || node.getExecLocation() == ExecLocation.Map) {
- N MRparent = getParentNode(node, execNodes, ExecLocation.MapAndReduce);
- if ( MRparent != null && MRparent.getType() == lopTypes[jobi]) {
- int numParents = node.getOutputs().size();
- if (numParents > 1) {
- for (int j = 0; j < numParents; j++) {
- if (!finishedNodes.contains(node.getOutputs()
- .get(j)))
- nodesWithUnfinishedOutputs.add(node);
- }
-
- }
- }
- }
- /*
- // Following condition will not work for MMRJ because execNodes may contain non-MapAndReduce
- // lops that are compatible with MMRJ (e.g., Data-WRITE)
- else if (!(node.getExecLocation() == ExecLocation.MapAndReduce
- && node.getType() == lopTypes[jobi])) {
- throw new LopsException(
- "Error in handleSingleOutputJobs(): encountered incompatible execLocation='"
- + node.getExecLocation() + "' for lop (ID="
- + node.getID() + ").");
- }
- */
- }
-
- // need to redo all nodes in nodesWithOutput as well as their
- // children
-
- for (int i = 0; i < vec.size(); i++) {
- N node = vec.get(i);
- if (node.getExecLocation() == ExecLocation.MapOrReduce
- || node.getExecLocation() == ExecLocation.Map) {
- if (nodesWithUnfinishedOutputs.contains(node))
- finishedNodes.remove(node);
-
- if (hasParentNode(node, nodesWithUnfinishedOutputs))
- finishedNodes.remove(node);
-
- }
- }
- }
- }
-
- }
-
- /** Method to check if a lop can be eliminated from checking **/
- private boolean canEliminateLop(N node, ArrayList<N> execNodes) {
- // this function can only eliminate "aligner" lops such a group
- if (!node.isAligner())
- return false;
-
- // find the child whose execLoc = 'MapAndReduce'
- int ret = getChildAlignment(node, execNodes, ExecLocation.MapAndReduce);
-
- if (ret == CHILD_BREAKS_ALIGNMENT)
- return false;
- else if (ret == CHILD_DOES_NOT_BREAK_ALIGNMENT)
- return true;
- else if (ret == MRCHILD_NOT_FOUND)
- return false;
- else if (ret == MR_CHILD_FOUND_BREAKS_ALIGNMENT)
- return false;
- else if (ret == MR_CHILD_FOUND_DOES_NOT_BREAK_ALIGNMENT)
- return true;
- else
- throw new RuntimeException("Should not happen. \n");
- }
-
-
- /**
- * Method to generate createvar instructions, which creates a new entry
- * in the symbol table. One instruction is generated for every LOP that is
- * 1) type Data and
- * 2) persistent and
- * 3) matrix and
- * 4) read
- *
- * Transient reads needn't be considered here since the previous program
- * block would already create appropriate entries in the symbol table.
- *
- * @param nodes
- * @throws LopsException
- */
- private void generateInstructionsForInputVariables(ArrayList<N> nodes_v, ArrayList<Instruction> inst) throws LopsException, IOException {
- for(N n : nodes_v) {
- if (n.getExecLocation() == ExecLocation.Data && !((Data) n).isTransient()
- && ((Data) n).getOperationType() == OperationTypes.READ
- && (n.getDataType() == DataType.MATRIX || n.getDataType() == DataType.FRAME) ) {
-
- if ( !((Data)n).isLiteral() ) {
- try {
- String inst_string = n.getInstructions();
- CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(inst_string);
- currInstr.setLocation(n);
- inst.add(currInstr);
- } catch (DMLUnsupportedOperationException e) {
- throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
- } catch (DMLRuntimeException e) {
- throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
- }
- }
- }
- }
- }
-
- /*private String getOutputFormat(Data node) {
- if(node.getFileFormatType() == FileFormatTypes.TEXT)
- return "textcell";
- else if ( node.getOutputParameters().isBlocked_representation() )
- return "binaryblock";
- else
- return "binarycell";
- }*/
-
- /**
- * Determine whether to send <code>node</code> to MR or to process it in the control program.
- * It is sent to MR in the following cases:
- *
- * 1) if input lop gets processed in MR then <code>node</code> can be piggybacked
- *
- * 2) if the exectype of write lop itself is marked MR i.e., memory estimate > memory budget.
- *
- * @param node
- * @return
- */
- @SuppressWarnings("unchecked")
- private boolean sendWriteLopToMR(N node)
- {
- if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE )
- return false;
- N in = (N) node.getInputs().get(0);
- Format nodeFormat = node.getOutputParameters().getFormat();
-
- // Case of a transient read feeding into only one output persistent binaryblock write
- // Move the temporary file on HDFS to required persistent location, insteadof copying.
- if ( in.getExecLocation() == ExecLocation.Data && in.getOutputs().size() == 1
- && !((Data)node).isTransient()
- && ((Data)in).isTransient()
- && ((Data)in).getOutputParameters().isBlocked()
- && node.getOutputParameters().isBlocked() ) {
- return false;
- }
-
- //send write lop to MR if (1) it is marked with exec type MR (based on its memory estimate), or
- //(2) if the input lop is in MR and the write format allows to pack it into the same job (this does
- //not apply to csv write because MR csvwrite is a separate MR job type)
- if( node.getExecType() == ExecType.MR || (in.getExecType() == ExecType.MR && nodeFormat != Format.CSV ) )
- return true;
- else
- return false;
- }
-
- /**
- * Computes the memory footprint required to execute <code>node</code> in the mapper.
- * It is used only for those nodes that use inputs from distributed cache. The returned
- * value is utilized in limiting the number of instructions piggybacked onto a single GMR mapper.
- */
- private double computeFootprintInMapper(N node) {
- // Memory limits must be checked only for nodes that use distributed cache
- if ( ! node.usesDistributedCache() )
- // default behavior
- return 0.0;
-
- OutputParameters in1dims = node.getInputs().get(0).getOutputParameters();
- OutputParameters in2dims = node.getInputs().get(1).getOutputParameters();
-
- double footprint = 0;
- if ( node instanceof MapMult ) {
- int dcInputIndex = node.distributedCacheInputIndex()[0];
- footprint = AggBinaryOp.getMapmmMemEstimate(
- in1dims.getNumRows(), in1dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(),
- in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(),
- dcInputIndex, false);
- }
- else if ( node instanceof PMMJ ) {
- int dcInputIndex = node.distributedCacheInputIndex()[0];
- footprint = AggBinaryOp.getMapmmMemEstimate(
- in1dims.getNumRows(), 1, in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(),
- in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(),
- dcInputIndex, true);
- }
- else if ( node instanceof AppendM ) {
- footprint = BinaryOp.footprintInMapper(
- in1dims.getNumRows(), in1dims.getNumCols(),
- in2dims.getNumRows(), in2dims.getNumCols(),
- in1dims.getRowsInBlock(), in1dims.getColsInBlock());
- }
- else if ( node instanceof BinaryM ) {
- footprint = BinaryOp.footprintInMapper(
- in1dims.getNumRows(), in1dims.getNumCols(),
- in2dims.getNumRows(), in2dims.getNumCols(),
- in1dims.getRowsInBlock(), in1dims.getColsInBlock());
- }
- else {
- // default behavior
- return 0.0;
- }
- return footprint;
- }
-
- /**
- * Determines if <code>node</code> can be executed in current round of MR jobs or if it needs to be queued for later rounds.
- * If the total estimated footprint (<code>node</code> and previously added nodes in GMR) is less than available memory on
- * the mappers then <code>node</code> can be executed in current round, and <code>true</code> is returned. Otherwise,
- * <code>node</code> must be queued and <code>false</code> is returned.
- */
- private boolean checkMemoryLimits(N node, double footprintInMapper) {
- boolean addNode = true;
-
- // Memory limits must be checked only for nodes that use distributed cache
- if ( ! node.usesDistributedCache() )
- // default behavior
- return addNode;
-
- double memBudget = Math.min(AggBinaryOp.MAPMULT_MEM_MULTIPLIER, BinaryOp.APPEND_MEM_MULTIPLIER) * OptimizerUtils.getRemoteMemBudgetMap(true);
- if ( footprintInMapper <= memBudget )
- return addNode;
- else
- return !addNode;
- }
-
- /**
- * Method to group a vector of sorted lops.
- *
- * @param node_v
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
-
- @SuppressWarnings("unchecked")
- private ArrayList<Instruction> doGreedyGrouping(StatementBlock sb, ArrayList<N> node_v)
- throws LopsException, IOException, DMLRuntimeException,
- DMLUnsupportedOperationException
-
- {
- LOG.trace("Grouping DAG ============");
-
- // nodes to be executed in current iteration
- ArrayList<N> execNodes = new ArrayList<N>();
- // nodes that have already been processed
- ArrayList<N> finishedNodes = new ArrayList<N>();
- // nodes that are queued for the following iteration
- ArrayList<N> queuedNodes = new ArrayList<N>();
-
- ArrayList<ArrayList<N>> jobNodes = createNodeVectors(JobType.getNumJobTypes());
-
-
- // list of instructions
- ArrayList<Instruction> inst = new ArrayList<Instruction>();
-
- //ArrayList<Instruction> preWriteDeleteInst = new ArrayList<Instruction>();
- ArrayList<Instruction> writeInst = new ArrayList<Instruction>();
- ArrayList<Instruction> deleteInst = new ArrayList<Instruction>();
- ArrayList<Instruction> endOfBlockInst = new ArrayList<Instruction>();
-
- // delete transient variables that are no longer needed
- //deleteUnwantedTransientReadVariables(node_v, deleteInst);
-
- // remove files for transient reads that are updated.
- deleteUpdatedTransientReadVariables(sb, node_v, writeInst);
-
- generateRemoveInstructions(sb, endOfBlockInst);
-
- generateInstructionsForInputVariables(node_v, inst);
-
-
- boolean done = false;
- String indent = " ";
-
- while (!done) {
- LOG.trace("Grouping nodes in DAG");
-
- execNodes.clear();
- queuedNodes.clear();
- clearNodeVectors(jobNodes);
- gmrMapperFootprint=0;
-
- for (int i = 0; i < node_v.size(); i++) {
- N node = node_v.get(i);
-
- // finished nodes don't need to be processed
-
- if (finishedNodes.contains(node))
- continue;
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Processing node (" + node.getID()
- + ") " + node.toString() + " exec nodes size is " + execNodes.size());
-
-
- //if node defines MR job, make sure it is compatible with all
- //its children nodes in execNodes
- if(node.definesMRJob() && !compatibleWithChildrenInExecNodes(execNodes, node))
- {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing node "
- + node.toString() + " (code 1)");
-
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
- continue;
- }
-
- // if child is queued, this node will be processed in the later
- // iteration
- if (hasChildNode(node,queuedNodes)) {
-
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing node "
- + node.toString() + " (code 2)");
- //for(N q: queuedNodes) {
- // LOG.trace(indent + " " + q.getType() + "," + q.getID());
- //}
- queuedNodes.add(node);
-
- // if node has more than two inputs,
- // remove children that will be needed in a future
- // iterations
- // may also have to remove parent nodes of these children
- removeNodesForNextIteration(node, finishedNodes, execNodes,
- queuedNodes, jobNodes);
-
- continue;
- }
-
- // if inputs come from different jobs, then queue
- if ( node.getInputs().size() >= 2) {
- int jobid = Integer.MIN_VALUE;
- boolean queueit = false;
- for(int idx=0; idx < node.getInputs().size(); idx++) {
- int input_jobid = jobType(node.getInputs().get(idx), jobNodes);
- if (input_jobid != -1) {
- if ( jobid == Integer.MIN_VALUE )
- jobid = input_jobid;
- else if ( jobid != input_jobid ) {
- queueit = true;
- break;
- }
- }
- }
- if ( queueit ) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing node " + node.toString() + " (code 3)");
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
- continue;
- }
- }
-
- /*if (node.getInputs().size() == 2) {
- int j1 = jobType(node.getInputs().get(0), jobNodes);
- int j2 = jobType(node.getInputs().get(1), jobNodes);
- if (j1 != -1 && j2 != -1 && j1 != j2) {
- LOG.trace(indent + "Queueing node "
- + node.toString() + " (code 3)");
-
- queuedNodes.add(node);
-
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
-
- continue;
- }
- }*/
-
- // See if this lop can be eliminated
- // This check is for "aligner" lops (e.g., group)
- boolean eliminate = false;
- eliminate = canEliminateLop(node, execNodes);
- if (eliminate) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, eliminate);
- continue;
- }
-
- // If the node defines a MR Job then make sure none of its
- // children that defines a MR Job are present in execNodes
- if (node.definesMRJob()) {
- if (hasMRJobChildNode(node, execNodes)) {
- // "node" must NOT be queued when node=group and the child that defines job is Rand
- // this is because "group" can be pushed into the "Rand" job.
- if (! (node.getType() == Lop.Type.Grouping && checkDataGenAsChildNode(node,execNodes)) ) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing node " + node.toString() + " (code 4)");
-
- queuedNodes.add(node);
-
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
-
- continue;
- }
- }
- }
-
- // if "node" has more than one input, and has a descendant lop
- // in execNodes that is of type RecordReader
- // then all its inputs must be ancestors of RecordReader. If
- // not, queue "node"
- if (node.getInputs().size() > 1
- && hasChildNode(node, execNodes, ExecLocation.RecordReader)) {
- // get the actual RecordReader lop
- N rr_node = getChildNode(node, execNodes, ExecLocation.RecordReader);
-
- // all inputs of "node" must be ancestors of rr_node
- boolean queue_it = false;
- for (int in = 0; in < node.getInputs().size(); in++) {
- // each input should be ancestor of RecordReader lop
- N n = (N) node.getInputs().get(in);
- if (!n.equals(rr_node) && !isChild(rr_node, n, IDMap)) {
- queue_it = true; // i.e., "node" must be queued
- break;
- }
- }
-
- if (queue_it) {
- // queue node
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -" + node.toString() + " (code 5)");
- queuedNodes.add(node);
- // TODO: does this have to be modified to handle
- // recordreader lops?
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
- continue;
- } else {
- // nothing here.. subsequent checks have to be performed
- // on "node"
- ;
- }
- }
-
- // data node, always add if child not queued
- // only write nodes are kept in execnodes
- if (node.getExecLocation() == ExecLocation.Data) {
- Data dnode = (Data) node;
- boolean dnode_queued = false;
-
- if ( dnode.getOperationType() == OperationTypes.READ ) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding Data -"+ node.toString());
-
- // TODO: avoid readScalar instruction, and read it on-demand just like the way Matrices are read in control program
- if ( node.getDataType() == DataType.SCALAR
- //TODO: LEO check the following condition is still needed
- && node.getOutputParameters().getFile_name() != null ) {
- // this lop corresponds to reading a scalar from HDFS file
- // add it to execNodes so that "readScalar" instruction gets generated
- execNodes.add(node);
- // note: no need to add it to any job vector
- }
- }
- else if (dnode.getOperationType() == OperationTypes.WRITE) {
- // Skip the transient write <code>node</code> if the input is a
- // transient read with the same variable name. i.e., a dummy copy.
- // Hence, <code>node</code> can be avoided.
- // TODO: this case should ideally be handled in the language layer
- // prior to the construction of Hops Dag
- N input = (N) dnode.getInputs().get(0);
- if ( dnode.isTransient()
- && input.getExecLocation() == ExecLocation.Data
- && ((Data)input).isTransient()
- && dnode.getOutputParameters().getLabel().compareTo(input.getOutputParameters().getLabel()) == 0 ) {
- // do nothing, <code>node</code> must not processed any further.
- ;
- }
- else if ( execNodes.contains(input) && !isCompatible(node, input) && sendWriteLopToMR(node)) {
- // input is in execNodes but it is not compatible with write lop. So, queue the write lop.
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -" + node.toString());
- queuedNodes.add(node);
- dnode_queued = true;
- }
- else {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding Data -"+ node.toString());
-
- execNodes.add(node);
- if ( sendWriteLopToMR(node) ) {
- addNodeByJobType(node, jobNodes, execNodes, false);
- }
- }
- }
- if (!dnode_queued)
- finishedNodes.add(node);
-
- continue;
- }
-
- // map or reduce node, can always be piggybacked with parent
- if (node.getExecLocation() == ExecLocation.MapOrReduce) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, false);
-
- continue;
- }
-
- // RecordReader node, add, if no parent needs reduce, else queue
- if (node.getExecLocation() == ExecLocation.RecordReader) {
- // "node" should not have any children in
- // execNodes .. it has to be the first one in the job!
- if (!hasChildNode(node, execNodes, ExecLocation.Map)
- && !hasChildNode(node, execNodes,
- ExecLocation.MapAndReduce)) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, false);
- } else {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -"+ node.toString() + " (code 6)");
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
-
- }
- continue;
- }
-
- // map node, add, if no parent needs reduce, else queue
- if (node.getExecLocation() == ExecLocation.Map) {
- boolean queueThisNode = false;
- int subcode = -1;
- if ( node.usesDistributedCache() ) {
- // if an input to <code>node</code> comes from distributed cache
- // then that input must get executed in one of the previous jobs.
- int[] dcInputIndexes = node.distributedCacheInputIndex();
- for( int dcInputIndex : dcInputIndexes ){
- N dcInput = (N) node.getInputs().get(dcInputIndex-1);
- if ( (dcInput.getType() != Lop.Type.Data && dcInput.getExecType()==ExecType.MR)
- && execNodes.contains(dcInput) )
- {
- queueThisNode = true;
- subcode = 1;
- }
- }
-
- // Limit the number of distributed cache inputs based on the available memory in mappers
- double memsize = computeFootprintInMapper(node);
- //gmrMapperFootprint += computeFootprintInMapper(node);
- if ( gmrMapperFootprint>0 && !checkMemoryLimits(node, gmrMapperFootprint+memsize ) ) {
- queueThisNode = true;
- subcode = 2;
- }
- if(!queueThisNode)
- gmrMapperFootprint += memsize;
- }
- if (!queueThisNode && !hasChildNode(node, execNodes,ExecLocation.MapAndReduce)&& !hasMRJobChildNode(node, execNodes)) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, false);
- } else {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -"+ node.toString() + " (code 7 - " + "subcode " + subcode + ")");
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
-
- }
- continue;
- }
-
- // reduce node, make sure no parent needs reduce, else queue
- if (node.getExecLocation() == ExecLocation.MapAndReduce) {
-
- // boolean eliminate = false;
- // eliminate = canEliminateLop(node, execNodes);
- // if (eliminate || (!hasChildNode(node, execNodes,
- // ExecLocation.MapAndReduce)) &&
- // !hasMRJobChildNode(node,execNodes)) {
-
- // TODO: statiko -- keep the middle condition
- // discuss about having a lop that is MapAndReduce but does
- // not define a job
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, eliminate);
-
- // } else {
- // if (DEBUG)
- // System.out.println("Queueing -" + node.toString());
- // queuedNodes.add(node);
- // removeNodesForNextIteration(node, finishedNodes,
- // execNodes, queuedNodes, jobNodes);
- // }
- continue;
- }
-
- // aligned reduce, make sure a parent that is reduce exists
- if (node.getExecLocation() == ExecLocation.Reduce) {
- if ( compatibleWithChildrenInExecNodes(execNodes, node) &&
- (hasChildNode(node, execNodes, ExecLocation.MapAndReduce)
- || hasChildNode(node, execNodes, ExecLocation.Map) ) )
- {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding -"+ node.toString());
- execNodes.add(node);
- finishedNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, false);
- } else {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -"+ node.toString() + " (code 8)");
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
- }
-
- continue;
-
- }
-
- // add Scalar to execNodes if it has no child in exec nodes
- // that will be executed in a MR job.
- if (node.getExecLocation() == ExecLocation.ControlProgram) {
- for (int j = 0; j < node.getInputs().size(); j++) {
- if (execNodes.contains(node.getInputs().get(j))
- && !(node.getInputs().get(j).getExecLocation() == ExecLocation.Data)
- && !(node.getInputs().get(j).getExecLocation() == ExecLocation.ControlProgram)) {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -"+ node.toString() + " (code 9)");
-
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
- break;
- }
- }
-
- if (queuedNodes.contains(node))
- continue;
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Adding - scalar"+ node.toString());
- execNodes.add(node);
- addNodeByJobType(node, jobNodes, execNodes, false);
- finishedNodes.add(node);
- continue;
- }
-
- }
-
- // no work to do
- if ( execNodes.isEmpty() ) {
-
- if( !queuedNodes.isEmpty() )
- {
- //System.err.println("Queued nodes should be 0");
- throw new LopsException("Queued nodes should not be 0 at this point \n");
- }
-
- if( LOG.isTraceEnabled() )
- LOG.trace("All done! queuedNodes = "+ queuedNodes.size());
-
- done = true;
- } else {
- // work to do
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Generating jobs for group -- Node count="+ execNodes.size());
-
- // first process scalar instructions
- generateControlProgramJobs(execNodes, inst, writeInst, deleteInst);
-
- // copy unassigned lops in execnodes to gmrnodes
- for (int i = 0; i < execNodes.size(); i++) {
- N node = execNodes.get(i);
- if (jobType(node, jobNodes) == -1) {
- if ( isCompatible(node, JobType.GMR) ) {
- if ( node.hasNonBlockedInputs() ) {
- jobNodes.get(JobType.GMRCELL.getId()).add(node);
- addChildren(node, jobNodes.get(JobType.GMRCELL.getId()), execNodes);
- }
- else {
- jobNodes.get(JobType.GMR.getId()).add(node);
- addChildren(node, jobNodes.get(JobType.GMR.getId()), execNodes);
- }
- }
- else {
- if( LOG.isTraceEnabled() )
- LOG.trace(indent + "Queueing -" + node.toString() + " (code 10)");
- execNodes.remove(i);
- finishedNodes.remove(node);
- queuedNodes.add(node);
- removeNodesForNextIteration(node, finishedNodes,
- execNodes, queuedNodes, jobNodes);
- }
- }
- }
-
- // next generate MR instructions
- if (!execNodes.isEmpty())
- generateMRJobs(execNodes, inst, writeInst, deleteInst, jobNodes);
-
- handleSingleOutputJobs(execNodes, jobNodes, finishedNodes);
-
- }
-
- }
-
- // add write and delete inst at the very end.
-
- //inst.addAll(preWriteDeleteInst);
- inst.addAll(writeInst);
- inst.addAll(deleteInst);
- inst.addAll(endOfBlockInst);
-
- return inst;
-
- }
-
- private boolean compatibleWithChildrenInExecNodes(ArrayList<N> execNodes, N node) {
- for(int i=0; i < execNodes.size(); i++)
- {
- N tmpNode = execNodes.get(i);
- // for lops that execute in control program, compatibleJobs property is set to LopProperties.INVALID
- // we should not consider such lops in this check
- if (isChild(tmpNode, node, IDMap)
- && tmpNode.getExecLocation() != ExecLocation.ControlProgram
- //&& tmpNode.getCompatibleJobs() != LopProperties.INVALID
- && (tmpNode.getCompatibleJobs() & node.getCompatibleJobs()) == 0)
- return false;
- }
- return true;
- }
-
- /**
- * Exclude rmvar instruction for <varname> from deleteInst, if exists
- *
- * @param varName
- * @param deleteInst
- */
- private void excludeRemoveInstruction(String varName, ArrayList<Instruction> deleteInst) {
- //for(Instruction inst : deleteInst) {
- for(int i=0; i < deleteInst.size(); i++) {
- Instruction inst = deleteInst.get(i);
- if ((inst.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM || inst.getType() == INSTRUCTION_TYPE.SPARK)
- && ((CPInstruction)inst).getCPInstructionType() == CPINSTRUCTION_TYPE.Variable
- && ((VariableCPInstruction)inst).isRemoveVariable(varName) ) {
- deleteInst.remove(i);
- }
- }
- }
-
- /**
- * Generate rmvar instructions for the inputs, if their consumer count becomes zero.
- *
- * @param node
- * @param inst
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- private void processConsumersForInputs(N node, ArrayList<Instruction> inst, ArrayList<Instruction> delteInst) throws DMLRuntimeException, DMLUnsupportedOperationException {
- // reduce the consumer count for all input lops
- // if the count becomes zero, then then variable associated w/ input can be removed
- for(Lop in : node.getInputs() ) {
- if(DMLScript.ENABLE_DEBUG_MODE) {
- processConsumers((N)in, inst, delteInst, node);
- }
- else {
- processConsumers((N)in, inst, delteInst, null);
- }
-
- /*if ( in.removeConsumer() == 0 ) {
- String label = in.getOutputParameters().getLabel();
- inst.add(VariableCPInstruction.prepareRemoveInstruction(label));
- }*/
- }
- }
-
- private void processConsumers(N node, ArrayList<Instruction> inst, ArrayList<Instruction> deleteInst, N locationInfo) throws DMLRuntimeException, DMLUnsupportedOperationException {
- // reduce the consumer count for all input lops
- // if the count becomes zero, then then variable associated w/ input can be removed
- if ( node.removeConsumer() == 0 ) {
- if ( node.getExecLocation() == ExecLocation.Data && ((Data)node).isLiteral() ) {
- return;
- }
-
- String label = node.getOutputParameters().getLabel();
- Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(label);
- if (locationInfo != null)
- currInstr.setLocation(locationInfo);
- else
- currInstr.setLocation(node);
-
- inst.add(currInstr);
- excludeRemoveInstruction(label, deleteInst);
- }
- }
-
- /**
- * Method to generate instructions that are executed in Control Program. At
- * this point, this DAG has no dependencies on the MR dag. ie. none of the
- * inputs are outputs of MR jobs
- *
- * @param execNodes
- * @param inst
- * @param deleteInst
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
-
- @SuppressWarnings("unchecked")
- private void generateControlProgramJobs(ArrayList<N> execNodes,
- ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) throws LopsException,
- DMLUnsupportedOperationException, DMLRuntimeException {
-
- // nodes to be deleted from execnodes
- ArrayList<N> markedNodes = new ArrayList<N>();
-
- // variable names to be deleted
- ArrayList<String> var_deletions = new ArrayList<String>();
- HashMap<String, N> var_deletionsLineNum = new HashMap<String, N>();
-
- boolean doRmVar = false;
-
- for (int i = 0; i < execNodes.size(); i++) {
- N node = execNodes.get(i);
- doRmVar = false;
-
- // mark input scalar read nodes for deletion
- // TODO: statiko -- check if this condition ever evaluated to TRUE
- if (node.getExecLocation() == ExecLocation.Data
- && ((Data) node).getOperationType() == Data.OperationTypes.READ
- && ((Data) node).getDataType() == DataType.SCALAR
- && node.getOutputParameters().getFile_name() == null ) {
- markedNodes.add(node);
- continue;
- }
-
- // output scalar instructions and mark nodes for deletion
- if (node.getExecLocation() == ExecLocation.ControlProgram) {
-
- if (node.getDataType() == DataType.SCALAR) {
- // Output from lops with SCALAR data type must
- // go into Temporary Variables (Var0, Var1, etc.)
- NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
- inst.addAll(out.getPreInstructions()); // dummy
- deleteInst.addAll(out.getLastInstructions());
- } else {
- // Output from lops with non-SCALAR data type must
- // go into Temporary Files (temp0, temp1, etc.)
-
- NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
- inst.addAll(out.getPreInstructions());
-
- boolean hasTransientWriteParent = false;
- for ( int pid=0; pid < node.getOutputs().size(); pid++ ) {
- N parent = (N)node.getOutputs().get(pid);
- if ( parent.getExecLocation() == ExecLocation.Data
- && ((Data)parent).getOperationType() == Data.OperationTypes.WRITE
- && ((Data)parent).isTransient() ) {
- hasTransientWriteParent = true;
- break;
- }
- }
-
- if ( !hasTransientWriteParent ) {
- deleteInst.addAll(out.getLastInstructions());
- }
- else {
- var_deletions.add(node.getOutputParameters().getLabel());
- var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
-
- //System.out.println(" --> skipping " + out.getLastInstructions() + " while processing node " + node.getID());
- }
- }
-
- String inst_string = "";
-
- // Lops with arbitrary number of inputs (ParameterizedBuiltin, GroupedAggregate, DataGen)
- // are handled separately, by simply passing ONLY the output variable to getInstructions()
- if (node.getType() == Lop.Type.ParameterizedBuiltin
- || node.getType() == Lop.Type.GroupedAgg
- || node.getType() == Lop.Type.DataGen ){
- inst_string = node.getInstructions(node.getOutputParameters().getLabel());
- }
-
- // Lops with arbitrary number of inputs and outputs are handled
- // separately as well by passing arrays of inputs and outputs
- else if ( node.getType() == Lop.Type.FunctionCallCP )
- {
- String[] inputs = new String[node.getInputs().size()];
- String[] outputs = new String[node.getOutputs().size()];
- int count = 0;
- for( Lop in : node.getInputs() )
- inputs[count++] = in.getOutputParameters().getLabel();
- count = 0;
- for( Lop out : node.getOutputs() )
- {
- outputs[count++] = out.getOutputParameters().getLabel();
- }
-
- inst_string = node.getInstructions(inputs, outputs);
- }
- else {
- if ( node.getInputs().isEmpty() ) {
- // currently, such a case exists only for Rand lop
- inst_string = node.getInstructions(node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 1) {
- inst_string = node.getInstructions(node.getInputs()
- .get(0).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 2) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 3 || node.getType() == Type.Ternary) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getInputs().get(2).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 4) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getInputs().get(2).getOutputParameters().getLabel(),
- node.getInputs().get(3).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 5) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getInputs().get(2).getOutputParameters().getLabel(),
- node.getInputs().get(3).getOutputParameters().getLabel(),
- node.getInputs().get(4).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 6) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getInputs().get(2).getOutputParameters().getLabel(),
- node.getInputs().get(3).getOutputParameters().getLabel(),
- node.getInputs().get(4).getOutputParameters().getLabel(),
- node.getInputs().get(5).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
- else if (node.getInputs().size() == 7) {
- inst_string = node.getInstructions(
- node.getInputs().get(0).getOutputParameters().getLabel(),
- node.getInputs().get(1).getOutputParameters().getLabel(),
- node.getInputs().get(2).getOutputParameters().getLabel(),
- node.getInputs().get(3).getOutputParameters().getLabel(),
- node.getInputs().get(4).getOutputParameters().getLabel(),
- node.getInputs().get(5).getOutputParameters().getLabel(),
- node.getInputs().get(6).getOutputParameters().getLabel(),
- node.getOutputParameters().getLabel());
- }
-
- else {
- throw new LopsException(node.printErrorLocation() + "Node with " + node.getInputs().size() + " inputs is not supported in CP yet! \n");
- }
- }
-
- try {
- if( LOG.isTraceEnabled() )
- LOG.trace("Generating instruction - "+ inst_string);
- Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
- if (node._beginLine != 0)
- currInstr.setLocation(node);
- else if ( !node.getOutputs().isEmpty() )
- currInstr.setLocation(node.getOutputs().get(0));
- else if ( !node.getInputs().isEmpty() )
- currInstr.setLocation(node.getInputs().get(0));
-
- inst.add(currInstr);
- } catch (Exception e) {
- throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - "
- + inst_string, e);
- }
-
- markedNodes.add(node);
- doRmVar = true;
- //continue;
- }
- else if (node.getExecLocation() == ExecLocation.Data ) {
- Data dnode = (Data)node;
- Data.OperationTypes op = dnode.getOperationType();
-
- if ( op == Data.OperationTypes.WRITE ) {
- NodeOutput out = null;
- if ( sendWriteLopToMR(node) ) {
- // In this case, Data WRITE lop goes into MR, and
- // we don't have to do anything here
- doRmVar = false;
- }
- else {
- out = setupNodeOutputs(node, ExecType.CP, false, false);
- if ( dnode.getDataType() == DataType.SCALAR ) {
- // processing is same for both transient and persistent scalar writes
- writeInst.addAll(out.getLastInstructions());
- //inst.addAll(out.getLastInstructions());
- doRmVar = false;
- }
- else {
- // setupNodeOutputs() handles both transient and persistent matrix writes
- if ( dnode.isTransient() ) {
- //inst.addAll(out.getPreInstructions()); // dummy ?
- deleteInst.addAll(out.getLastInstructions());
- doRmVar = false;
- }
- else {
- // In case of persistent write lop, write instruction will be generated
- // and that instruction must be added to <code>inst</code> so that it gets
- // executed immediately. If it is added to <code>deleteInst</code> then it
- // gets executed at the end of program block's execution
- inst.addAll(out.getLastInstructions());
- doRmVar = true;
- }
- }
- markedNodes.add(node);
- //continue;
- }
- }
- else {
- // generate a temp label to hold the value that is read from HDFS
- if ( node.getDataType() == DataType.SCALAR ) {
- node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
- String io_inst = node.getInstructions(node.getOutputParameters().getLabel(),
- node.getOutputParameters().getFile_name());
- CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
- currInstr.setLocation(node);
-
- inst.add(currInstr);
-
- Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
- tempInstr.setLocation(node);
- deleteInst.add(tempInstr);
- }
- else {
- throw new LopsException("Matrix READs are not handled in CP yet!");
- }
- markedNodes.add(node);
- doRmVar = true;
- //continue;
- }
- }
-
- // see if rmvar instructions can be generated for node's inputs
- if(doRmVar)
- processConsumersForInputs(node, inst, deleteInst);
- doRmVar = false;
- }
-
- for ( String var : var_deletions ) {
- Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
- if( LOG.isTraceEnabled() )
- LOG.trace(" Adding var_deletions: " + rmInst.toString());
-
- rmInst.setLocation(var_deletionsLineNum.get(var));
-
- deleteInst.add(rmInst);
- }
-
- // delete all marked nodes
- for (int i = 0; i < markedNodes.size(); i++) {
- execNodes.remove(markedNodes.get(i));
- }
-
- }
-
- /**
- * Method to remove all child nodes of a queued node that should be executed
- * in a following iteration.
- *
- * @param node
- * @param finishedNodes
- * @param execNodes
- * @param queuedNodes
- * @throws LopsException
- */
-
- private void removeNodesForNextIteration(N node, ArrayList<N> finishedNodes,
- ArrayList<N> execNodes, ArrayList<N> queuedNodes,
- ArrayList<ArrayList<N>> jobvec) throws LopsException {
-
- // only queued nodes with multiple inputs need to be handled.
- if (node.getInputs().size() == 1)
- return;
-
- //if all children are queued, then there is nothing to do.
- int numInputs = node.getInputs().size();
- boolean allQueued = true;
- for(int i=0; i < numInputs; i++) {
- if( !queuedNodes.contains(node.getInputs().get(i)) ) {
- allQueued = false;
- break;
- }
- }
- if ( allQueued )
- return;
-
- if( LOG.isTraceEnabled() )
- LOG.trace(" Before remove nodes for next iteration -- size of execNodes " + execNodes.size());
-
- // Determine if <code>node</code> has inputs from the same job or multiple jobs
- int jobid = Integer.MIN_VALUE;
- boolean inputs_in_same_job = true;
- for(int idx=0; idx < node.getInputs().size(); idx++) {
- int input_jobid = jobType(node.getInputs().get(idx), jobvec);
- if ( jobid == Integer.MIN_VALUE )
- jobid = input_jobid;
- else if ( jobid != input_jobid ) {
- inputs_in_same_job = false;
- break;
- }
- }
-
- // Determine if there exist any unassigned inputs to <code>node</code>
- // Evaluate only those lops that execute in MR.
- boolean unassigned_inputs = false;
- for(int i=0; i < numInputs; i++) {
- Lop input = node.getInputs().get(i);
- //if ( input.getExecLocation() != ExecLocation.ControlProgram && jobType(input, jobvec) == -1 ) {
- if ( input.getExecType() == ExecType.MR && !execNodes.contains(input)) { //jobType(input, jobvec) == -1 ) {
- unassigned_inputs = true;
- break;
- }
- }
-
- // Determine if any node's children are queued
- boolean child_queued = false;
- for(int i=0; i < numInputs; i++) {
- if (queuedNodes.contains(node.getInputs().get(i)) ) {
- child_queued = true;
- break;
- }
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace(" Property Flags:");
- LOG.trace(" Inputs in same job: " + inputs_in_same_job);
- LOG.trace(" Unassigned inputs: " + unassigned_inputs);
- LOG.trace(" Child queued: " + child_queued);
- }
-
- // Evaluate each lop in <code>execNodes</code> for removal.
- // Add lops to be removed to <code>markedNodes</code>.
-
- ArrayList<N> markedNodes = new ArrayList<N>();
- for (int i = 0; i < execNodes.size(); i++) {
-
- N tmpNode = execNodes.get(i);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace(" Checking for removal (" + tmpNode.getID() + ") " + tmpNode.toString());
- }
-
- // if tmpNode is not a descendant of 'node', then there is no advantage in removing tmpNode for later iterations.
- if(!isChild(tmpNode, node, IDMap))
- continue;
-
- // handle group input lops
- if(node.getInputs().contains(tmpNode) && tmpNode.isAligner()) {
- markedNodes.add(tmpNode);
- if( LOG.isTraceEnabled() )
- LOG.trace(" Removing for next iteration (code 1): (" + tmpNode.getID() + ") " + tmpNode.toString());
- }
-
- //if (child_queued) {
- // if one of the children are queued,
- // remove some child nodes on other leg that may be needed later on.
- // For e.g. Group lop.
-
- if (!hasOtherQueuedParentNode(tmpNode, queuedNodes, node)
- && branchHasNoOtherUnExecutedParents(tmpNode, node, execNodes, finishedNodes)) {
-
- boolean queueit = false;
- int code = -1;
- switch(node.getExecLocation()) {
- case Map:
- if(branchCanBePiggyBackedMap(tmpNode, node, execNodes, queuedNodes, markedNodes))
- queueit = true;
- code=2;
- break;
-
- case MapAndReduce:
- if(branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, queuedNodes)&& !tmpNode.definesMRJob())
- queueit = true;
- code=3;
- break;
- case Reduce:
- if(branchCanBePiggyBackedReduce(tmpNode, node, execNodes, queuedNodes))
- queueit = true;
- code=4;
- break;
- default:
- //do nothing
- }
-
- if(queueit) {
- if( LOG.isTraceEnabled() )
- LOG.trace(" Removing for next iteration (code " + code + "): (" + tmpNode.getID() + ") " + tmpNode.toString());
-
- markedNodes.add(tmpNode);
- }
- }
- /*
- * "node" has no other queued children.
- *
- * If inputs are in the same job and "node" is of type
- * MapAndReduce, then remove nodes of all types other than
- * Reduce, MapAndReduce, and the ones that define a MR job as
- * they can be piggybacked later.
- *
- * e.g: A=Rand, B=Rand, C=A%*%B Here, both inputs of MMCJ lop
- * come from Rand job, and they should not be removed.
- *
- * Other examples: -- MMCJ whose children are of type
- * MapAndReduce (say GMR) -- Inputs coming from two different
- * jobs .. GMR & REBLOCK
- */
- //boolean himr = hasOtherMapAndReduceParentNode(tmpNode, execNodes,node);
- //boolean bcbp = branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, finishedNodes);
- //System.out.println(" .. " + inputs_in_same_job + "," + himr + "," + bcbp);
- if ((inputs_in_same_job || unassigned_inputs)
- && node.getExecLocation() == ExecLocation.MapAndReduce
- && !hasOtherMapAndReduceParentNode(tmpNode, execNodes,node) // don't remove since it already piggybacked with a MapReduce node
- && branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, queuedNodes)
- && !tmpNode.definesMRJob()) {
- if( LOG.isTraceEnabled() )
- LOG.trace(" Removing for next iteration (code 5): ("+ tmpNode.getID() + ") " + tmpNode.toString());
-
- markedNodes.add(tmpNode);
- }
- } // for i
-
- // we also need to delete all parent nodes of marked nodes
- for (int i = 0; i < execNodes.size(); i++) {
- LOG.trace(" Checking for removal - ("
- + execNodes.get(i).getID() + ") "
- + execNodes.get(i).toString());
-
- if (hasChildNode(execNodes.get(i), markedNodes) && !markedNodes.contains(execNodes.get(i))) {
- markedNodes.add(execNodes.get(i));
- LOG.trace(" Removing for next iteration (code 6) (" + execNodes.get(i).getID() + ") " + execNodes.get(i).toString());
- }
- }
-
- if ( execNodes.size() != markedNodes.size() ) {
- // delete marked nodes from finishedNodes and execNodes
- // add to queued nodes
- for(N n : markedNodes) {
- if ( n.usesDistributedCache() )
- gmrMapperFootprint -= computeFootprintInMapper(n);
- finishedNodes.remove(n);
- execNodes.remove(n);
- removeNodeByJobType(n, jobvec);
- queuedNodes.add(n);
- }
- }
- /*for (int i = 0; i < markedNodes.size(); i++) {
- finishedNodes.remove(markedNodes.elementAt(i));
- execNodes.remove(markedNodes.elementAt(i));
- removeNodeByJobType(markedNodes.elementAt(i), jobvec);
- queuedNodes.add(markedNodes.elementAt(i));
- }*/
- }
-
- @SuppressWarnings("unused")
- private void removeNodesForNextIterationOLD(N node, ArrayList<N> finishedNodes,
- ArrayList<N> execNodes, ArrayList<N> queuedNodes,
- ArrayList<ArrayList<N>> jobvec) throws LopsException {
- // only queued nodes with two inputs need to be handled.
-
- // TODO: statiko -- this should be made == 1
- if (node.getInputs().size() != 2)
- return;
-
-
- //if both children are queued, nothing to do.
- if (queuedNodes.contains(node.getInputs().get(0))
- && queuedNodes.contains(node.getInputs().get(1)))
- return;
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Before remove nodes for next iteration -- size of execNodes " + execNodes.size());
-
-
-
- boolean inputs_in_same_job = false;
- // TODO: handle tertiary
- if (jobType(node.getInputs().get(0), jobvec) != -1
- && jobType(node.getInputs().get(0), jobvec) == jobType(node
- .getInputs().get(1), jobvec)) {
- inputs_in_same_job = true;
- }
-
- boolean unassigned_inputs = false;
- // here.. scalars shd be ignored
- if ((node.getInputs().get(0).getExecLocation() != ExecLocation.ControlProgram && jobType(
- node.getInputs().get(0), jobvec) == -1)
- || (node.getInputs().get(1).getExecLocation() != ExecLocation.ControlProgram && jobType(
- node.getInputs().get(1), jobvec) == -1)) {
- unassigned_inputs = true;
- }
-
- boolean child_queued = false;
-
- // check if atleast one child was queued.
- if (queuedNodes.contains(node.getInputs().get(0))
- || queuedNodes.contains(node.getInputs().get(1)))
- child_queued = true;
-
- // nodes to be dropped
- ArrayList<N> markedNodes = new ArrayList<N>();
-
- for (int i = 0; i < execNodes.size(); i++) {
-
- N tmpNode = execNodes.get(i);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("Checking for removal (" + tmpNode.getID()
- + ") " + tmpNode.toString());
- LOG.trace("Inputs in same job " + inputs_in_same_job);
- LOG.trace("Unassigned inputs " + unassigned_inputs);
- LOG.trace("Child queued " + child_queued);
-
- }
-
- //if(node.definesMRJob() && isChild(tmpNode,node) && (tmpNode.getCompatibleJobs() & node.getCompatibleJobs()) == 0)
- // continue;
-
- // TODO: statiko -- check if this is too conservative?
- if (child_queued) {
- // if one of the children are queued,
- // remove some child nodes on other leg that may be needed later on.
- // For e.g. Group lop.
-
- if((tmpNode == node.getInputs().get(0) || tmpNode == node.getInputs().get(1)) &&
- tmpNode.isAligner())
- {
- markedNodes.add(tmpNode);
- if( LOG.isTraceEnabled() )
- LOG.trace("Removing for next iteration: ("
- + tmpNode.getID() + ") " + tmpNode.toString());
- }
- else {
- if (!hasOtherQueuedParentNode(tmpNode, queuedNodes, node)
- && isChild(tmpNode, node, IDMap) && branchHasNoOtherUnExecutedParents(tmpNode, node, execNodes, finishedNodes)) {
-
-
- if(
- //e.g. MMCJ
- (node.getExecLocation() == ExecLocation.MapAndReduce &&
- branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, finishedNodes) && !tmpNode.definesMRJob() )
- ||
- //e.g. Binary
- (node.getExecLocation() == ExecLocation.Reduce && branchCanBePiggyBackedReduce(tmpNode, node, execNodes, finishedNodes)) )
- {
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Removing for next iteration: ("
- + tmpNode.getID() + ") " + tmpNode.toString());
-
-
- markedNodes.add(tmpNode);
- }
- }
- }
- } else {
- /*
- * "node" has no other queued children.
- *
- * If inputs are in the same job and "node" is of type
- * MapAndReduce, then remove nodes of all types other than
- * Reduce, MapAndReduce, and the ones that define a MR job as
- * they can be piggybacked later.
- *
- * e.g: A=Rand, B=Rand, C=A%*%B Here, both inputs of MMCJ lop
- * come from Rand job, and they should not be removed.
- *
- * Other examples: -- MMCJ whose children are of type
- * MapAndReduce (say GMR) -- Inputs coming from two different
- * jobs .. GMR & REBLOCK
- */
- if ((inputs_in_same_job || unassigned_inputs)
- && node.getExecLocation() == ExecLocation.MapAndReduce
- && !hasOtherMapAndReduceParentNode(tmpNode, execNodes,
- node)
- && isChild(tmpNode, node, IDMap) &&
- branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, finishedNodes)
- && !tmpNode.definesMRJob()) {
- if( LOG.isTraceEnabled() )
- LOG.trace("Removing for next iteration:: ("
- + tmpNode.getID() + ") " + tmpNode.toString());
-
- markedNodes.add(tmpNode);
- }
-
- // as this node has inputs coming from different jobs, need to
- // free up everything
- // below and include the closest MapAndReduce lop if this is of
- // type Reduce.
- // if it is of type MapAndReduce, don't need to free any nodes
-
- if (!inputs_in_same_job && !unassigned_inputs
- && isChild(tmpNode, node, IDMap) &&
- (tmpNode == node.getInputs().get(0) || tmpNode == node.getInputs().get(1)) &&
- tmpNode.isAligner())
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Removing for next iteration ("
- + tmpNode.getID()
- + ") "
- + tmpNode.toString());
-
- markedNodes.add(tmpNode);
-
- }
-
- }
- }
-
- // we also need to delete all parent nodes of marked nodes
- for (int i = 0; i < execNodes.size(); i++) {
- if( LOG.isTraceEnabled() )
- LOG.trace("Checking for removal - ("
- + execNodes.get(i).getID() + ") "
- + execNodes.get(i).toString());
-
- if (hasChildNode(execNodes.get(i), markedNodes)) {
- markedNodes.add(execNodes.get(i));
- if( LOG.isTraceEnabled() )
- LOG.trace("Removing for next iteration - ("
- + execNodes.get(i).getID() + ") "
- + execNodes.get(i).toString());
- }
- }
-
- // delete marked nodes from finishedNodes and execNodes
- // add to queued nodes
- for (int i = 0; i < markedNodes.size(); i++) {
- finishedNodes.remove(markedNodes.get(i));
- execNodes.remove(markedNodes.get(i));
- removeNodeByJobType(markedNodes.get(i), jobvec);
- queuedNodes.add(markedNodes.get(i));
- }
- }
-
- private boolean branchCanBePiggyBackedReduce(N tmpNode, N node, ArrayList<N> execNodes, ArrayList<N> queuedNodes) {
- if(node.getExecLocation() != ExecLocation.Reduce)
- return false;
-
- // if tmpNode is descendant of any queued child of node, then branch can not be piggybacked
- for(Lop ni : node.getInputs()) {
- if(queuedNodes.contains(ni) && isChild(tmpNode, ni, IDMap))
- return false;
- }
-
- for(int i=0; i < execNodes.size(); i++) {
- N n = execNodes.get(i);
-
- if(n.equals(node))
- continue;
-
- if(n.equals(tmpNode) && n.getExecLocation() != ExecLocation.Map && n.getExecLocation() != ExecLocation.MapOrReduce)
- return false;
-
- // check if n is on the branch tmpNode->*->node
- if(isChild(n, node, IDMap) && isChild(tmpNode, n, IDMap)) {
- if(!node.getInputs().contains(tmpNode) // redundant
- && n.getExecLocation() != ExecLocation.Map && n.getExecLocation() != ExecLocation.MapOrReduce)
- return false;
- }
- /*if(isChild(n, node, IDMap) && isChild(tmpNode, n, IDMap)
- && !node.getInputs().contains(tmpNode)
- && n.getExecLocation() != ExecLocation.Map && n.getExecLocation() != ExecLocation.MapOrReduce)
- return false;*/
- }
- return true;
- }
-
- @SuppressWarnings("unchecked")
- private boolean branchCanBePiggyBackedMap(N tmpNode, N node, ArrayList<N> execNodes, ArrayList<N> queuedNodes, ArrayList<N> markedNodes) {
- if(node.getExecLocation() != ExecLocation.Map)
- return false;
-
- // if tmpNode is descendant of any queued child of node, then branch can not be piggybacked
- for(Lop ni : node.getInputs()) {
- if(queuedNodes != null && queuedNodes.contains(ni) && isChild(tmpNode, ni, IDMap))
- return false;
- }
-
- // since node.location=Map: only Map & MapOrReduce lops must be considered
- if( tmpNode.definesMRJob() || (tmpNode.getExecLocation() != ExecLocation.Map && tmpNode.getExecLocation() != ExecLocation.MapOrReduce))
- return false;
-
- // if there exist a node "dcInput" that is
- // -- a) parent of tmpNode, and b) feeds into "node" via distributed cache
- // then, tmpNode should not be removed.
- // "dcInput" must be executed prior to "node", and removal of tmpNode does not make that happen.
- if(node.usesDistributedCache() ) {
- for(int dcInputIndex : node.distributedCacheInputIndex()) {
- N dcInput = (N) node.getInputs().get(dcInputIndex-1);
- if(isChild(tmpNode, dcInput, IDMap))
- return false;
- }
- }
-
- // if tmpNode requires an input from distributed cache,
- // remove tmpNode only if that input can fit into mappers' memory. If not,
- if ( tmpNode.usesDistributedCache() ) {
- double memsize = computeFootprintInMapper(tmpNode);
- if (node.usesDistributedCache() )
- memsize += computeFootprintInMapper(node);
- if ( markedNodes != null ) {
- for(N n : markedNodes) {
- if ( n.usesDistributedCache() )
- memsize += computeFootprintInMapper(n);
- }
- }
- if ( !checkMemoryLimits(node, memsize ) ) {
- return false;
- }
- }
-
- if( (tmpNode.getCompatibleJobs() & node.getCompatibleJobs()) > 0)
- return true;
- else
- return false;
-
- }
-
- /**
- * Function that checks if <code>tmpNode</code> can be piggybacked with MapAndReduce
- * lop <code>node</code>.
- *
- * Decision depends on the exec location of <code>tmpNode</code>. If the exec location is:
- * MapAndReduce: CAN NOT be piggybacked since it defines its own MR job
- * Reduce: CAN NOT be piggybacked since it must execute before <code>node</code>
- * Map or MapOrReduce: CAN be piggybacked ONLY IF it is comatible w/ <code>tmpNode</code>
- *
- * @param tmpNode
- * @param node
- * @param execNodes
- * @param finishedNodes
- * @return
- */
- private boolean branchCanBePiggyBackedMapAndReduce(N tmpNode, N node,
- ArrayList<N> execNodes, ArrayList<N> queuedNodes) {
-
- if (node.getExecLocation() != ExecLocation.MapAndReduce)
- return false;
- JobType jt = JobType.findJobTypeFromLop(node);
-
- for (int i = 0; i < execNodes.size(); i++) {
- N n = execNodes.get(i);
-
- if (n.equals(node))
- continue;
-
- // Evaluate only nodes on the branch between tmpNode->..->node
- if (n.equals(tmpNode) || (isChild(n, node, IDMap) && isChild(tmpNode, n, IDMap))) {
- if ( hasOtherMapAndReduceParentNode(tmpNode, queuedNodes,node) )
- return false;
- ExecLocation el = n.getExecLocation();
- if (el != ExecLocation.Map && el != ExecLocation.MapOrReduce)
- return false;
- else if (!isCompatible(n, jt))
- return false;
- }
-
- /*
- * if(n.equals(tmpNode) && n.getExecLocation() != ExecLocation.Map
- * && n.getExecLocation() != ExecLocation.MapOrReduce) return false;
- *
- * if(isChild(n, node, IDMap) && isChild(tmpNode, n, IDMap) &&
- * n.getExecLocation() != ExecLocation.Map && n.getExecLocation() !=
- * ExecLocation.MapOrReduce) return false;
- */
-
- }
- return true;
- }
-
- private boolean branchHasNoOtherUnExecutedParents(N tmpNode, N node,
- ArrayList<N> execNodes, ArrayList<N> finishedNodes) {
-
- //if tmpNode has more than one unfinished output, return false
- if(tmpNode.getOutputs().size() > 1)
- {
- int cnt = 0;
- for (int j = 0; j < tmpNode.getOutputs().size(); j++) {
- if (!finishedNodes.contains(tmpNode.getOutputs().get(j)))
- cnt++;
- }
-
- if(cnt != 1)
- return false;
- }
-
- //check to see if any node between node and tmpNode has more than one unfinished output
- for(int i=0; i < execNodes.size(); i++)
- {
- N n = execNodes.get(i);
-
- if(n.equals(node) || n.equals(tmpNode))
- continue;
-
- if(isChild(n, node, IDMap) && isChild(tmpNode, n, IDMap))
- {
- int cnt = 0;
- for (int j = 0; j < n.getOutputs().size(); j++) {
- if (!finishedNodes.contains(n.getOutputs().get(j)))
- cnt++;
- }
-
- if(cnt != 1)
- return false;
- }
- }
-
- return true;
- }
-
- /**
- * Method to check of there is a node of type MapAndReduce between a child
- * (tmpNode) and its parent (node)
- *
- * @param tmpNode
- * @param execNodes
- * @param node
- * @return
- */
-
- @SuppressWarnings({ "unchecked", "unused" })
- private boolean hasMapAndReduceParentNode(N tmpNode, ArrayList<N> execNodes, N node)
- {
- for (int i = 0; i < tmpNode.getOutputs().size(); i++) {
- N n = (N) tmpNode.getOutputs().get(i);
-
- if (execNodes.contains(n)
- && n.getExecLocation() == ExecLocation.MapAndReduce
- && isChild(n, node, IDMap)) {
- return true;
- } else {
- if (hasMapAndReduceParentNode(n, execNodes, node))
- return true;
- }
-
- }
-
- return false;
- }
-
- /**
- * Method to return the job index for a lop.
- *
- * @param lops
- * @param jobvec
- * @return
- * @throws LopsException
- */
-
- private int jobType(Lop lops, ArrayList<ArrayList<N>> jobvec) throws LopsException {
- for ( JobType jt : JobType.values()) {
- int i = jt.getId();
- if (i > 0 && jobvec.get(i) != null && jobvec.get(i).contains(lops)) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Method to see if there is a node of type MapAndReduce between tmpNode and node
- * in given node collection
- *
- * @param tmpNode
- * @param nodeList
- * @param node
- * @return
- */
-
- @SuppressWarnings("unchecked")
- private boolean hasOtherMapAndReduceParentNode(N tmpNode,
- ArrayList<N> nodeList, N node) {
-
- if ( tmpNode.getExecLocation() == ExecLocation.MapAndReduce)
- return true;
-
- for (int i = 0; i < tmpNode.getOutputs().size(); i++) {
- N n = (N) tmpNode.getOutputs().get(i);
-
- if ( nodeList.contains(n) && isChild(n,node,IDMap)) {
- if(!n.equals(node) && n.getExecLocation() == ExecLocation.MapAndReduce)
- return true;
- else
- return hasOtherMapAndReduceParentNode(n, nodeList, node);
- }
- }
-
- return false;
- }
-
- /**
- * Method to check if there is a queued node that is a parent of both tmpNode and node
- *
- * @param tmpNode
- * @param queuedNodes
- * @param node
- * @return
- */
- private boolean hasOtherQueuedParentNode(N tmpNode, ArrayList<N> queuedNodes, N node) {
- if ( queuedNodes.isEmpty() )
- return false;
-
- boolean[] nodeMarked = node.get_reachable();
- boolean[] tmpMarked = tmpNode.get_reachable();
- long nodeid = IDMap.get(node.getID());
- long tmpid = IDMap.get(tmpNode.getID());
-
- for ( int i=0; i < queuedNodes.size(); i++ ) {
- int id = IDMap.get(queuedNodes.get(i).getID());
- if ((id != nodeid && nodeMarked[id]) && (id != tmpid && tmpMarked[id]) )
- //if (nodeMarked[id] && tmpMarked[id])
- return true;
- }
-
- return false;
- }
-
- /**
- * Method to print the lops grouped by job type
- *
- * @param jobNodes
- * @throws DMLRuntimeException
- */
- void printJobNodes(ArrayList<ArrayList<N>> jobNodes)
- throws DMLRuntimeException {
- if (LOG.isTraceEnabled()){
- for ( JobType jt : JobType.values() ) {
- int i = jt.getId();
- if (i > 0 && jobNodes.get(i) != null && !jobNodes.get(i).isEmpty() ) {
- LOG.trace(jt.getName() + " Job Nodes:");
-
- for (int j = 0; j < jobNodes.get(i).size(); j++) {
- LOG.trace(" "
- + jobNodes.get(i).get(j).getID() + ") "
- + jobNodes.get(i).get(j).toString());
- }
- }
- }
-
- }
-
-
- }
-
- /**
- * Method to check if there exists any lops with ExecLocation=RecordReader
- *
- * @param nodes
- * @param loc
- * @return
- */
- boolean hasANode(ArrayList<N> nodes, ExecLocation loc) {
- for (int i = 0; i < nodes.size(); i++) {
- if (nodes.get(i).getExecLocation() == ExecLocation.RecordReader)
- return true;
- }
- return false;
- }
-
- ArrayList<ArrayList<N>> splitGMRNodesByRecordReader(ArrayList<N> gmrnodes) {
-
- // obtain the list of record reader nodes
- ArrayList<N> rrnodes = new ArrayList<N>();
- for (int i = 0; i < gmrnodes.size(); i++) {
- if (gmrnodes.get(i).getExecLocation() == ExecLocation.RecordReader)
- rrnodes.add(gmrnodes.get(i));
- }
-
- /*
- * We allocate one extra vector to hold lops that do not depend on any
- * recordreader lops
- */
- ArrayList<ArrayList<N>> splitGMR = createNodeVectors(rrnodes.size() + 1);
-
- // flags to indicate whether a lop has been added to one of the node
- // vectors
- boolean[] flags = new boolean[gmrnodes.size()];
- for (int i = 0; i < gmrnodes.size(); i++) {
- flags[i] = false;
- }
-
- // first, obtain all ancestors of recordreader lops
- for (int rrid = 0; rrid < rrnodes.size(); rrid++) {
- // prepare node list for i^th record reader lop
-
- // add record reader lop
- splitGMR.get(rrid).add(rrnodes.get(rrid));
-
- for (int j = 0; j < gmrnodes.size(); j++) {
- if (rrnodes.get(rrid).equals(gmrnodes.get(j)))
- flags[j] = true;
- else if (isChild(rrnodes.get(rrid), gmrnodes.get(j), IDMap)) {
- splitGMR.get(rrid).add(gmrnodes.get(j));
- flags[j] = true;
- }
- }
- }
-
- // add all remaining lops to a separate job
- int jobindex = rrnodes.size(); // the last node vector
- for (int i = 0; i < gmrnodes.size(); i++) {
- if (!flags[i]) {
- splitGMR.get(jobindex).add(gmrnodes.get(i));
- flags[i] = true;
- }
- }
-
- return splitGMR;
- }
-
- /**
- * Method to generate hadoop jobs. Exec nodes can contains a mixture of node
- * types requiring different mr jobs. This method breaks the job into
- * sub-types and then invokes the appropriate method to generate
- * instructions.
- *
- * @param execNodes
- * @param inst
- * @param deleteinst
- * @param
<TRUNCATED>
[19/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DataIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DataIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/DataIdentifier.java
deleted file mode 100644
index 709be3f..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DataIdentifier.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-public class DataIdentifier extends Identifier
-{
-
- protected String _name;
- protected String _valueTypeString;
- protected String _defaultValue;
-
- public DataIdentifier(DataIdentifier passed){
- setProperties(passed);
- _kind = Kind.Data;
- _name = passed.getName();
- _valueTypeString = passed.getValueType().toString();
- _defaultValue = passed.getDefaultValue();
-
- // set location information
- setFilename(passed.getFilename());
- setBeginLine(passed.getBeginLine());
- setBeginColumn(passed.getBeginColumn());
- setEndLine(passed.getEndLine());
- setEndColumn(passed.getEndColumn());
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- DataIdentifier newId = new DataIdentifier(this);
- String newIdName = prefix + this._name;
- newId.setName(newIdName);
-
- return newId;
- }
-
- public DataIdentifier(String name){
- super();
- _name = name;
- _kind = Kind.Data;
- _defaultValue = null;
-
- }
-
- /*
- public DataIdentifier(String name, int line, int col){
- super();
- _name = name;
- _kind = Kind.Data;
- _defaultValue = null;
- }
- */
- public DataIdentifier(){
- _name = null;
- _kind = null;
- _defaultValue = null;
- }
-
-
- public void setTypeInfo( String valueType, String dataType) throws ParseException{
-
- if (valueType.equalsIgnoreCase("int") || valueType.equalsIgnoreCase("integer"))
- this.setValueType(ValueType.INT);
- else if (valueType.equalsIgnoreCase("double"))
- this.setValueType(ValueType.DOUBLE);
- else if (valueType.equalsIgnoreCase("string"))
- this.setValueType(ValueType.STRING);
- else if (valueType.equalsIgnoreCase("boolean"))
- this.setValueType(ValueType.BOOLEAN);
- else if (valueType.equalsIgnoreCase("object"))
- this.setValueType(ValueType.OBJECT);
- else {
- // provide location for this exception in the parser
- LOG.error(this.printErrorLocation() + "function parameter has unknown value type " + valueType);
- throw new ParseException(this.printErrorLocation() + "function parameter has unknown value type " + valueType);
- }
-
- if (dataType.equalsIgnoreCase("object"))
- this.setDataType(DataType.OBJECT);
- else if (dataType.equalsIgnoreCase("SCALAR"))
- this.setDataType(DataType.SCALAR);
- else if (dataType.equalsIgnoreCase("MATRIX"))
- this.setDataType(DataType.MATRIX);
- else {
- // provide location for this exception in the parser
- LOG.error(this.printErrorLocation() + "function parameter has unknown data type " + valueType);
- throw new ParseException(this.printErrorLocation() + "function parameter has unknown data type " + valueType);
- }
-
- }
-
- public String getName(){
- return _name;
- }
- public void setName(String name){
- _name = name;
- }
- public String getDefaultValue(){
- return _defaultValue;
- }
- public void setDefaultValue(String val){
- _defaultValue = val;
- }
-
- @Override
- public String toString() {
- return _name;
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariable(_name, this);
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- /**
- * Method to specify if an expression returns multiple outputs.
- * This method must be overridden by all child classes.
- * @return
- */
- public boolean multipleReturns() throws LanguageException {
- throw new LanguageException("multipleReturns() must be overridden in the subclass.");
- }
-
- @Override
- public boolean equals(Object that)
- {
- if( !(that instanceof DataIdentifier) )
- return false;
-
- DataIdentifier target = (DataIdentifier)that;
- if(getName()!=null && !getName().equals(target.getName()))
- return false;
- if(getDataType()!=null && !getDataType().equals(target.getDataType()))
- return false;
- if(getValueType() != null && !getValueType().equals(target.getValueType()))
- return false;
- if(getFormatType()!= null && !this.getFormatType().equals(target.getFormatType()))
- return false;
- if(!(this.getDim1() == target.getDim1()))
- return false;
- if(!(this.getDim2() == target.getDim2()))
- return false;
-
- return true;
-
- }
-
- @Override
- public int hashCode()
- {
- return super.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DoubleIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DoubleIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/DoubleIdentifier.java
deleted file mode 100644
index 257f652..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DoubleIdentifier.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-
-public class DoubleIdentifier extends ConstIdentifier
-{
-
- private double _val;
-
-
- public DoubleIdentifier(double val, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = val;
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.DOUBLE);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public DoubleIdentifier(DoubleIdentifier d, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = d.getValue();
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.DOUBLE);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- return this;
- }
-
- // Used only by the parser for unary operation
- public void multiplyByMinusOne() {
- _val = -1 * _val;
- }
-
- public double getValue(){
- return _val;
- }
-
- public void setValue(double v) {
- _val = v;
- }
-
- public String toString(){
- return Double.toString(_val);
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public long getLongValue() {
- return UtilFunctions.toLong(getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/Expression.java b/src/main/java/com/ibm/bi/dml/parser/Expression.java
deleted file mode 100644
index 478b099..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/Expression.java
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-
-
-public abstract class Expression
-{
-
- public enum Kind {
- UnaryOp, BinaryOp, BooleanOp, BuiltinFunctionOp, ParameterizedBuiltinFunctionOp, DataOp, Data, Literal, RelationalOp, ExtBuiltinFunctionOp, FunctionCallOp
- };
-
- public enum BinaryOp {
- PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
- };
-
- public enum RelationalOp {
- LESSEQUAL, LESS, GREATEREQUAL, GREATER, EQUAL, NOTEQUAL, INVALID
- };
-
- public enum BooleanOp {
- CONDITIONALAND, CONDITIONALOR, LOGICALAND, LOGICALOR, NOT, INVALID
- };
-
- public enum BuiltinFunctionOp {
- ABS,
- ACOS,
- ASIN,
- ATAN,
- AVG,
- CAST_AS_MATRIX,
- CAST_AS_SCALAR,
- CAST_AS_DOUBLE,
- CAST_AS_INT,
- CAST_AS_BOOLEAN,
- COLMEAN,
- COLMAX,
- COLMIN,
- COLSUM,
- COS,
- COV,
- CUMMAX,
- CUMMIN,
- CUMPROD,
- CUMSUM,
- DIAG,
- EXP,
- INTERQUANTILE,
- IQM,
- LENGTH,
- LOG,
- MAX,
- MEAN,
- MIN,
- MOMENT,
- NCOL,
- NROW,
- OUTER,
- PPRED,
- PROD,
- QUANTILE,
- RANGE,
- ROUND,
- ROWINDEXMAX,
- ROWMAX,
- ROWMEAN,
- ROWMIN,
- ROWINDEXMIN,
- ROWSUM,
- SEQ,
- SIN,
- SQRT,
- SUM,
- TABLE,
- TAN,
- TRACE,
- TRANS,
- QR,
- LU,
- EIGEN,
- SOLVE,
- CEIL,
- FLOOR,
- CBIND, //previously APPEND
- RBIND,
- MEDIAN,
- INVERSE,
- SAMPLE
- };
-
- public enum ParameterizedBuiltinFunctionOp {
- GROUPEDAGG, RMEMPTY, REPLACE, ORDER,
- // Distribution Functions
- CDF, INVCDF, PNORM, QNORM, PT, QT, PF, QF, PCHISQ, QCHISQ, PEXP, QEXP,
- TRANSFORM,
- INVALID
- };
-
- public enum DataOp {
- READ, WRITE, RAND, MATRIX, INVALID
- }
-
- public enum FunctCallOp {
- INTERNAL, EXTERNAL
- };
-
- public enum ExtBuiltinFunctionOp {
- EIGEN, CHOLESKY
- };
-
- public enum AggOp {
- SUM, MIN, MAX, INVALID
- };
-
- public enum ReorgOp {
- TRANSPOSE, DIAG
- };
-
- //public enum DataOp {
- // READ, WRITE
- //};
-
- public enum DataType {
- MATRIX, SCALAR, FRAME, OBJECT, UNKNOWN;
-
- public boolean isMatrix() {
- return (this == MATRIX);
- }
- public boolean isScalar() {
- return (this == SCALAR);
- }
- };
-
- public enum ValueType {
- INT, DOUBLE, STRING, BOOLEAN, OBJECT, UNKNOWN
- };
-
- public enum FormatType {
- TEXT, BINARY, MM, CSV, UNKNOWN
- };
-
- protected static final Log LOG = LogFactory.getLog(Expression.class.getName());
-
- public abstract Expression rewriteExpression(String prefix) throws LanguageException;
-
-
- protected Kind _kind;
- protected Identifier[] _outputs;
-
- private static int _tempId;
-
- public Expression() {
- _outputs = null;
- }
-
- public void setOutput(Identifier output) {
- if ( _outputs == null) {
- _outputs = new Identifier[1];
- }
- _outputs[0] = output;
- }
-
- public Kind getKind() {
- return _kind;
- }
-
- public Identifier getOutput() {
- if (_outputs != null && _outputs.length > 0)
- return _outputs[0];
- else
- return null;
- }
-
- public Identifier[] getOutputs() {
- return _outputs;
- }
-
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
- throws LanguageException
- {
- raiseValidateError("Should never be invoked in Baseclass 'Expression'", false);
- }
-
- public void validateExpression(MultiAssignmentStatement mas, HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
- throws LanguageException
- {
- raiseValidateError("Should never be invoked in Baseclass 'Expression'", false);
- }
-
- public static BinaryOp getBinaryOp(String val) {
- if (val.equalsIgnoreCase("+"))
- return BinaryOp.PLUS;
- else if (val.equalsIgnoreCase("-"))
- return BinaryOp.MINUS;
- else if (val.equalsIgnoreCase("*"))
- return BinaryOp.MULT;
- else if (val.equalsIgnoreCase("/"))
- return BinaryOp.DIV;
- else if (val.equalsIgnoreCase("%%"))
- return BinaryOp.MODULUS;
- else if (val.equalsIgnoreCase("%/%"))
- return BinaryOp.INTDIV;
- else if (val.equalsIgnoreCase("^"))
- return BinaryOp.POW;
- else if (val.equalsIgnoreCase("%*%"))
- return BinaryOp.MATMULT;
- return BinaryOp.INVALID;
- }
-
- public static RelationalOp getRelationalOp(String val) {
- if (val == null)
- return null;
- else if (val.equalsIgnoreCase("<"))
- return RelationalOp.LESS;
- else if (val.equalsIgnoreCase("<="))
- return RelationalOp.LESSEQUAL;
- else if (val.equalsIgnoreCase(">"))
- return RelationalOp.GREATER;
- else if (val.equalsIgnoreCase(">="))
- return RelationalOp.GREATEREQUAL;
- else if (val.equalsIgnoreCase("=="))
- return RelationalOp.EQUAL;
- else if (val.equalsIgnoreCase("!="))
- return RelationalOp.NOTEQUAL;
- return RelationalOp.INVALID;
- }
-
- public static BooleanOp getBooleanOp(String val) {
- if (val.equalsIgnoreCase("&&"))
- return BooleanOp.CONDITIONALAND;
- else if (val.equalsIgnoreCase("&"))
- return BooleanOp.LOGICALAND;
- else if (val.equalsIgnoreCase("||"))
- return BooleanOp.CONDITIONALOR;
- else if (val.equalsIgnoreCase("|"))
- return BooleanOp.LOGICALOR;
- else if (val.equalsIgnoreCase("!"))
- return BooleanOp.NOT;
- return BooleanOp.INVALID;
- }
-
- /**
- * Convert format types from parser to Hops enum : default is text
- */
-
- public static FileFormatTypes convertFormatType(String fn) {
- if (fn == null)
- return FileFormatTypes.TEXT;
- if (fn.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_TEXT)) {
- return FileFormatTypes.TEXT;
- }
- if (fn.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_BINARY)) {
- return FileFormatTypes.BINARY;
- }
- if (fn.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET)) {
- return FileFormatTypes.MM;
- }
- if (fn.equalsIgnoreCase(DataExpression.FORMAT_TYPE_VALUE_CSV)) {
- return FileFormatTypes.CSV;
- }
- // ToDo : throw parse exception for invalid / unsupported format type
- return FileFormatTypes.TEXT;
- }
-
- /**
- * Construct Hops from parse tree : Create temporary views in expressions
- */
- public static String getTempName() {
- return "parsertemp" + _tempId++;
- }
-
- public abstract VariableSet variablesRead();
-
- public abstract VariableSet variablesUpdated();
-
- public static DataType computeDataType(Expression e1, Expression e2, boolean cast) throws LanguageException {
- return computeDataType(e1.getOutput(), e2.getOutput(), cast);
- }
-
- public static DataType computeDataType(Identifier id1, Identifier id2, boolean cast) throws LanguageException {
- DataType d1 = id1.getDataType();
- DataType d2 = id2.getDataType();
-
- if (d1 == d2)
- return d1;
-
- if (cast) {
- if (d1 == DataType.MATRIX && d2 == DataType.SCALAR)
- return DataType.MATRIX;
- if (d1 == DataType.SCALAR && d2 == DataType.MATRIX)
- return DataType.MATRIX;
- }
-
- //raise error with id1 location
- id1.raiseValidateError("Invalid Datatypes for operation "+d1+" "+d2, false,
- LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- return null; //never reached because unconditional
- }
-
- public static ValueType computeValueType(Expression e1, Expression e2, boolean cast) throws LanguageException {
- return computeValueType(e1.getOutput(), e2.getOutput(), cast);
- }
-
- public static ValueType computeValueType(Identifier id1, Identifier id2, boolean cast) throws LanguageException {
- ValueType v1 = id1.getValueType();
- ValueType v2 = id2.getValueType();
-
- if (v1 == v2)
- return v1;
-
- if (cast) {
- if (v1 == ValueType.DOUBLE && v2 == ValueType.INT)
- return ValueType.DOUBLE;
- if (v2 == ValueType.DOUBLE && v1 == ValueType.INT)
- return ValueType.DOUBLE;
-
- // String value type will override others
- // Primary operation involving strings is concatenation (+)
- if ( v1 == ValueType.STRING || v2 == ValueType.STRING )
- return ValueType.STRING;
- }
-
- //raise error with id1 location
- id1.raiseValidateError("Invalid Valuetypes for operation "+v1+" "+v2, false,
- LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- return null; //never reached because unconditional
- }
-
- @Override
- public boolean equals(Object that)
- {
- //empty check for robustness
- if( that == null || !(that instanceof Expression) )
- return false;
-
- Expression thatExpr = (Expression) that;
-
- //approach is to compare string representation of expression
- String thisStr = this.toString();
- String thatStr = thatExpr.toString();
-
- return thisStr.equalsIgnoreCase(thatStr);
- }
-
- @Override
- public int hashCode()
- {
- //use identity hash code
- return super.hashCode();
- }
-
- ///////////////////////////////////////////////////////////////
- // validate error handling (consistent for all expressions)
-
- /**
- *
- * @param msg
- * @param conditional
- * @throws LanguageException
- */
- public void raiseValidateError( String msg, boolean conditional )
- throws LanguageException
- {
- raiseValidateError(msg, conditional, null);
- }
-
- /**
- *
- * @param msg
- * @param conditional
- * @param code
- * @throws LanguageException
- */
- public void raiseValidateError( String msg, boolean conditional, String errorCode )
- throws LanguageException
- {
- if( conditional ) //warning if conditional
- {
- String fullMsg = this.printWarningLocation() + msg;
-
- LOG.warn( fullMsg );
- }
- else //error and exception if unconditional
- {
- String fullMsg = this.printErrorLocation() + msg;
-
- LOG.error( fullMsg );
- if( errorCode != null )
- throw new LanguageException( fullMsg, errorCode );
- else
- throw new LanguageException( fullMsg );
- }
- }
-
-
- /**
- * Returns the matrix characteristics for scalar-scalar, scalar-matrix, matrix-scalar, matrix-matrix
- * operations. This method is aware of potentially unknowns and matrix-vector (col/row) operations.
- *
- * Format: rlen, clen, brlen, bclen.
- *
- * @param left
- * @param right
- * @return
- */
- public static long[] getBinaryMatrixCharacteristics( Expression left, Expression right )
- {
- long[] ret = new long[]{ -1, -1, -1, -1 };
-
- Identifier idleft = left.getOutput();
- Identifier idright = right.getOutput();
-
- if( idleft.getDataType()==DataType.SCALAR && idright.getDataType()==DataType.SCALAR ) {
- ret[0] = 0;
- ret[1] = 0;
- ret[2] = 0;
- ret[3] = 0;
- }
- else if( idleft.getDataType()==DataType.SCALAR && idright.getDataType()==DataType.MATRIX ) {
- ret[0] = idright.getDim1();
- ret[1] = idright.getDim2();
- ret[2] = idright.getRowsInBlock();
- ret[3] = idright.getColumnsInBlock();
- }
- else if( idleft.getDataType()==DataType.MATRIX && idright.getDataType()==DataType.SCALAR ) {
- ret[0] = idleft.getDim1();
- ret[1] = idleft.getDim2();
- ret[2] = idleft.getRowsInBlock();
- ret[3] = idleft.getColumnsInBlock();
- }
- else if( idleft.getDataType()==DataType.MATRIX && idright.getDataType()==DataType.MATRIX ) {
- ret[0] = idleft.getDim1();
- ret[1] = idleft.getDim2();
- ret[2] = idleft.getRowsInBlock();
- ret[3] = idleft.getColumnsInBlock();
- if( ret[0] < 0 && idright.getDim1() > 1 ) //robustness for row vectors
- ret[0] = idright.getDim1();
- if( ret[1] < 0 && idright.getDim2() > 1 ) //robustness for row vectors
- ret[1] = idright.getDim2();
- }
-
- return ret;
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store exception info + position information for expressions
- ///////////////////////////////////////////////////////////////////////////
- private String _filename;
- private int _beginLine, _beginColumn;
- private int _endLine, _endColumn;
- private ArrayList<String> _parseExceptionList = new ArrayList<String>();
-
- public void setFilename(String passed) { _filename = passed; }
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
- public void setParseExceptionList(ArrayList<String> passed) { _parseExceptionList = passed;}
-
- public void setAllPositions(String filename, int blp, int bcp, int elp, int ecp){
- _filename = filename;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public String getFilename() { return _filename; }
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
- public ArrayList<String> getParseExceptionList() { return _parseExceptionList; }
-
- public String printErrorLocation(){
- return "ERROR: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printErrorLocation(int beginLine, int beginColumn){
- return "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printInfoLocation(){
- return "INFO: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ExternalFunctionStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ExternalFunctionStatement.java b/src/main/java/com/ibm/bi/dml/parser/ExternalFunctionStatement.java
deleted file mode 100644
index 65bd878..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ExternalFunctionStatement.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-
-
-public class ExternalFunctionStatement extends FunctionStatement
-{
-
- //valid attribute names
- public static final String CLASS_NAME = "classname";
- public static final String EXEC_TYPE = "exectype";
- //public static final String EXEC_LOCATION = "execlocation"; //MB: obsolete
- public static final String CONFIG_FILE = "configfile";
-
- //valid attribute values for execlocation and
- public static final String FILE_BASED = "file";
- public static final String IN_MEMORY = "mem";
- //public static final String MASTER = "master"; //MB: obsolete
- //public static final String WORKER = "worker"; //MB: obsolete
-
- //default values for optional attributes
- public static final String DEFAULT_EXEC_TYPE = FILE_BASED;
- //public static final String DEFAULT_EXEC_LOCATION = MASTER; //MB: obsolete
-
- //all parameters
- private HashMap<String,String> _otherParams;
-
-
- public ExternalFunctionStatement(){
- super();
- }
-
-
-
- public ExternalFunctionStatement( ArrayList<DataIdentifier> functionInputs, ArrayList<DataIdentifier> functionOutputs, HashMap<String,String> map)
- {
- super();
- _inputParams = functionInputs;
- _outputParams = (functionOutputs == null) ? new ArrayList<DataIdentifier>() : functionOutputs;
- setOtherParams(map);
- }
-
-
- public void setOtherParams(HashMap<String,String> params){
- _otherParams = params;
- }
-
- public HashMap<String,String> getOtherParams(){
- return _otherParams;
- }
-
- /**
- * Validates all attributes and attribute values.
- *
- * @throws LanguageException
- */
- public void validateParameters(StatementBlock sb) //always unconditional
- throws LanguageException
- {
-
- //warnings for all not defined attributes
- for( String varName : _otherParams.keySet() )
- if( !( varName.equals(CLASS_NAME) || varName.equals(EXEC_TYPE)
- || varName.equals(CONFIG_FILE) ) )
- {
- LOG.warn( printWarningLocation() + "External function specifies undefined attribute type '"+varName+"'.");
- }
-
- //class name (required)
- if( !_otherParams.containsKey(CLASS_NAME) ){
- sb.raiseValidateError("External function does not specify the required attribute '"+CLASS_NAME+"'.", false);
- }
- else if ( _otherParams.get(CLASS_NAME)==null ) {
- sb.raiseValidateError("External function specifies empty '"+CLASS_NAME+"'.", false);
- }
-
- //exec type (optional, default: file)
- if( _otherParams.containsKey( EXEC_TYPE ) )
- {
- //check specified values
- String execType = _otherParams.get(EXEC_TYPE);
- if( !(execType.equals(FILE_BASED) || execType.equals(IN_MEMORY)) ) { //always unconditional (invalid parameter)
- sb.raiseValidateError("External function specifies invalid value for (optional) attribute '"+EXEC_TYPE+"' (valid values: "+FILE_BASED+","+IN_MEMORY+").", false);
- }
- }
- else
- {
- //put default values
- _otherParams.put(EXEC_TYPE, DEFAULT_EXEC_TYPE);
- }
- }
-
- @Override
- public boolean controlStatement()
- {
- return true;
- }
-
- @Override
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(_name + " = ");
-
- sb.append("externalfunction ( ");
- for (int i=0; i<_inputParams.size(); i++){
- DataIdentifier curr = _inputParams.get(i);
- sb.append(curr.getName());
- if (curr.getDefaultValue() != null) sb.append(" = " + curr.getDefaultValue());
- if (i < _inputParams.size()-1) sb.append(", ");
- }
- sb.append(") return (");
-
- for (int i=0; i<_outputParams.size(); i++){
- sb.append(_outputParams.get(i).getName());
- if (i < _outputParams.size()-1) sb.append(", ");
- }
- sb.append(")\n implemented in (");
-
- int j = 0;
- for (String key : _otherParams.keySet()){
- sb.append(key + " = " + _otherParams.get(key));
- if (j < _otherParams.keySet().size()-1) sb.append(", ");
- j++;
- }
-
- sb.append(") \n");
- return sb.toString();
- }
-
- @Override
- public void initializeforwardLV(VariableSet activeIn) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for ExternalFunctionStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for ExternalFunctionStatement");
- }
-
- @Override
- public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for ExternalFunctionStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for ExternalFunctionStatement");
-
- }
-
- @Override
- public VariableSet variablesRead() {
- LOG.warn(this.printWarningLocation() + "should not call variablesRead from ExternalFunctionStatement ");
- return new VariableSet();
- }
-
- @Override
- public VariableSet variablesUpdated() {
- LOG.warn(this.printWarningLocation() + "should not call variablesRead from ExternalFunctionStatement ");
- return new VariableSet();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ForStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ForStatement.java b/src/main/java/com/ibm/bi/dml/parser/ForStatement.java
deleted file mode 100644
index b6c6ce4..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ForStatement.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-
-
-public class ForStatement extends Statement
-{
-
-
- protected IterablePredicate _predicate;
- protected ArrayList<StatementBlock> _body;
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should not call rewriteStatement for ForStatement");
- throw new LanguageException(this.printErrorLocation() + "should not call rewriteStatement for ForStatement");
- }
-
- public ForStatement(){
- _predicate = null;
- _body = new ArrayList<StatementBlock>();
- }
-
- public void setPredicate(IterablePredicate pred){
- _predicate = pred;
- }
-
-
- public void addStatementBlock(StatementBlock sb){
- _body.add(sb);
- }
-
- public IterablePredicate getIterablePredicate(){
- return _predicate;
- }
-
- public ArrayList<StatementBlock> getBody(){
- return _body;
- }
-
- public void setBody(ArrayList<StatementBlock> body){
- _body = body;
- }
-
-
- @Override
- public boolean controlStatement() {
- return true;
- }
-
- public void mergeStatementBlocks(){
- _body = StatementBlock.mergeStatementBlocks(_body);
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append("for ");
- sb.append(_predicate.toString());
- sb.append(" { \n");
- for (StatementBlock block : _body){
- sb.append(block.toString());
- }
- sb.append("}\n");
- return sb.toString();
- }
-
- public void initializeforwardLV(VariableSet activeIn) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for ForStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for ForStatement");
- }
-
- public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for ForStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for ForStatement");
-
- }
-
- @Override
- public VariableSet variablesRead() {
- LOG.error(this.printErrorLocation() + "should not call variablesRead from ForStatement ");
- return new VariableSet();
- }
-
- @Override
- public VariableSet variablesUpdated() {
- LOG.error(this.printErrorLocation() + "should not call variablesRead from ForStatement ");
- return new VariableSet();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ForStatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/ForStatementBlock.java
deleted file mode 100644
index 0e4c05c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ForStatementBlock.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-
-
-public class ForStatementBlock extends StatementBlock
-{
-
- protected Hop _fromHops = null;
- protected Hop _toHops = null;
- protected Hop _incrementHops = null;
-
- protected Lop _fromLops = null;
- protected Lop _toLops = null;
- protected Lop _incrementLops = null;
-
- protected boolean _requiresFromRecompile = false;
- protected boolean _requiresToRecompile = false;
- protected boolean _requiresIncrementRecompile = false;
-
-
- public IterablePredicate getIterPredicate(){
- return ((ForStatement)_statements.get(0)).getIterablePredicate();
- }
-
- @Override
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
- if (_statements.size() > 1){
- raiseValidateError("ForStatementBlock should have only 1 statement (for statement)", conditional);
- }
- ForStatement fs = (ForStatement) _statements.get(0);
- IterablePredicate predicate = fs.getIterablePredicate();
-
- // Record original size information before loop for ALL variables
- // Will compare size / type info for these after loop completes
- // Replace variables with changed size with unknown value
- VariableSet origVarsBeforeBody = new VariableSet();
- for (String key : ids.getVariableNames()){
- DataIdentifier origId = ids.getVariable(key);
- DataIdentifier copyId = new DataIdentifier(origId);
- origVarsBeforeBody.addVariable(key, copyId);
- }
-
- //////////////////////////////////////////////////////////////////////////////
- // FIRST PASS: process the predicate / statement blocks in the body of the for statement
- ///////////////////////////////////////////////////////////////////////////////
-
- //remove updated vars from constants
- for( String var : _updated.getVariableNames() )
- if( constVars.containsKey( var ) )
- constVars.remove( var );
-
- predicate.validateExpression(ids.getVariables(), constVars, conditional);
- ArrayList<StatementBlock> body = fs.getBody();
-
- //perform constant propagation for ( from, to, incr )
- //(e.g., useful for reducing false positives in parfor dependency analysis)
- performConstantPropagation(constVars);
-
- //validate body
- _dmlProg = dmlProg;
- for(StatementBlock sb : body)
- {
- ids = sb.validate(dmlProg, ids, constVars, true);
- constVars = sb.getConstOut();
- }
-
- if (!body.isEmpty()){
- _constVarsIn.putAll(body.get(0).getConstIn());
- _constVarsOut.putAll(body.get(body.size()-1).getConstOut());
- }
-
- // for each updated variable
- boolean revalidationRequired = false;
- for (String key : _updated.getVariableNames())
- {
- DataIdentifier startVersion = origVarsBeforeBody.getVariable(key);
- DataIdentifier endVersion = ids.getVariable(key);
-
- if (startVersion != null && endVersion != null)
- {
- //handle data type change (reject)
- if (!startVersion.getOutput().getDataType().equals(endVersion.getOutput().getDataType())){
- raiseValidateError("ForStatementBlock has unsupported conditional data type change of variable '"+key+"' in loop body.", conditional);
- }
-
- //handle size change
- long startVersionDim1 = (startVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)startVersion).getOrigDim1() : startVersion.getDim1();
- long endVersionDim1 = (endVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)endVersion).getOrigDim1() : endVersion.getDim1();
- long startVersionDim2 = (startVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)startVersion).getOrigDim2() : startVersion.getDim2();
- long endVersionDim2 = (endVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)endVersion).getOrigDim2() : endVersion.getDim2();
-
- boolean sizeUnchanged = ((startVersionDim1 == endVersionDim1) &&
- (startVersionDim2 == endVersionDim2) );
-
- //handle sparsity change
- //NOTE: nnz not propagated via validate, and hence, we conservatively assume that nnz have been changed.
- //long startVersionNNZ = startVersion.getNnz();
- //long endVersionNNZ = endVersion.getNnz();
- //boolean nnzUnchanged = (startVersionNNZ == endVersionNNZ);
- boolean nnzUnchanged = false;
-
- // IF size has changed --
- if (!sizeUnchanged || !nnzUnchanged){
- revalidationRequired = true;
- DataIdentifier recVersion = new DataIdentifier(endVersion);
- if(!sizeUnchanged)
- recVersion.setDimensions(-1, -1);
- if(!nnzUnchanged)
- recVersion.setNnz(-1);
- origVarsBeforeBody.addVariable(key, recVersion);
- }
- }
- }
-
- // revalidation is required -- size was updated for at least 1 variable
- if (revalidationRequired){
-
- // update ids to the reconciled values
- ids = origVarsBeforeBody;
-
- //////////////////////////////////////////////////////////////////////////////
- // SECOND PASS: process the predicate / statement blocks in the body of the for statement
- ///////////////////////////////////////////////////////////////////////////////
-
- //remove updated vars from constants
- for( String var : _updated.getVariableNames() )
- if( constVars.containsKey( var ) )
- constVars.remove( var );
-
- //perform constant propagation for ( from, to, incr )
- //(e.g., useful for reducing false positives in parfor dependency analysis)
- performConstantPropagation(constVars);
-
- predicate.validateExpression(ids.getVariables(), constVars, conditional);
- body = fs.getBody();
-
- //validate body
- _dmlProg = dmlProg;
- for(StatementBlock sb : body)
- {
- ids = sb.validate(dmlProg, ids, constVars, true);
- constVars = sb.getConstOut();
- }
- if (!body.isEmpty()){
- _constVarsIn.putAll(body.get(0).getConstIn());
- _constVarsOut.putAll(body.get(body.size()-1).getConstOut());
- }
- }
-
- return ids;
- }
-
- public VariableSet initializeforwardLV(VariableSet activeInPassed) throws LanguageException {
-
- ForStatement fstmt = (ForStatement)_statements.get(0);
- if (_statements.size() > 1){
- LOG.error(_statements.get(0).printErrorLocation() + "ForStatementBlock should have only 1 statement (for statement)");
- throw new LanguageException(_statements.get(0).printErrorLocation() + "ForStatementBlock should have only 1 statement (for statement)");
- }
-
- _read = new VariableSet();
- _read.addVariables(fstmt.getIterablePredicate().variablesRead());
- _updated.addVariables(fstmt.getIterablePredicate().variablesUpdated());
-
- _gen = new VariableSet();
- _gen.addVariables(fstmt.getIterablePredicate().variablesRead());
-
- // add the iterVar from iterable predicate to kill set
- _kill.addVariables(fstmt.getIterablePredicate().variablesUpdated());
-
- VariableSet current = new VariableSet();
- current.addVariables(activeInPassed);
- current.addVariables(_updated);
-
-
- for( StatementBlock sb : fstmt.getBody())
- {
- current = sb.initializeforwardLV(current);
-
- // for each generated variable in this block, check variable not killed
- // in prior statement block in while stmt blody
- for (String varName : sb._gen.getVariableNames()){
-
- // IF the variable is NOT set in the while loop PRIOR to this stmt block,
- // THEN needs to be generated
- if (!_kill.getVariableNames().contains(varName)){
- _gen.addVariable(varName, sb._gen.getVariable(varName));
- }
- }
-
- _read.addVariables(sb._read);
- _updated.addVariables(sb._updated);
-
- // only add kill variables for statement blocks guaranteed to execute
- if (!(sb instanceof WhileStatementBlock) && !(sb instanceof ForStatementBlock) ){
- _kill.addVariables(sb._kill);
- }
- }
-
- // set preliminary "warn" set -- variables that if used later may cause runtime error
- // if the loop is not executed
- // warnSet = (updated MINUS (updatedIfBody INTERSECT updatedElseBody)) MINUS current
- for (String varName : _updated.getVariableNames()){
- if (!activeInPassed.containsVariable(varName)) {
- _warnSet.addVariable(varName, _updated.getVariable(varName));
- }
- }
-
- // activeOut includes variables from passed live in and updated in the while body
- _liveOut = new VariableSet();
- _liveOut.addVariables(current);
- _liveOut.addVariables(_updated);
- return _liveOut;
- }
-
- public VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException{
-
- ForStatement fstmt = (ForStatement)_statements.get(0);
-
- VariableSet lo = new VariableSet();
- lo.addVariables(loPassed);
-
- // calls analyze for each statement block in while stmt body
- int numBlocks = fstmt.getBody().size();
- for (int i = numBlocks - 1; i >= 0; i--){
- lo = fstmt.getBody().get(i).analyze(lo);
- }
-
- VariableSet loReturn = new VariableSet();
- loReturn.addVariables(lo);
- return loReturn;
-
- }
-
- public ArrayList<Hop> get_hops() throws HopsException {
-
- if (_hops != null && !_hops.isEmpty()){
- LOG.error(this.printBlockErrorLocation() + "there should be no HOPs associated with the ForStatementBlock");
- throw new HopsException(this.printBlockErrorLocation() + "there should be no HOPs associated with the ForStatementBlock");
- }
-
- return _hops;
- }
-
- public void setFromHops(Hop hops) { _fromHops = hops; }
- public void setToHops(Hop hops) { _toHops = hops; }
- public void setIncrementHops(Hop hops) { _incrementHops = hops; }
-
- public Hop getFromHops() { return _fromHops; }
- public Hop getToHops() { return _toHops; }
- public Hop getIncrementHops() { return _incrementHops; }
-
- public void setFromLops(Lop lops) {
- _fromLops = lops;
- }
- public void setToLops(Lop lops) { _toLops = lops; }
- public void setIncrementLops(Lop lops) { _incrementLops = lops; }
-
- public Lop getFromLops() { return _fromLops; }
- public Lop getToLops() { return _toLops; }
- public Lop getIncrementLops() { return _incrementLops; }
-
-
-
- public VariableSet analyze(VariableSet loPassed) throws LanguageException{
-
- VariableSet predVars = new VariableSet();
- IterablePredicate ip = ((ForStatement)_statements.get(0)).getIterablePredicate();
-
- predVars.addVariables(ip.variablesRead());
- predVars.addVariables(ip.variablesUpdated());
-
- VariableSet candidateLO = new VariableSet();
- candidateLO.addVariables(loPassed);
- candidateLO.addVariables(_gen);
- candidateLO.addVariables(predVars);
-
- VariableSet origLiveOut = new VariableSet();
- origLiveOut.addVariables(_liveOut);
- origLiveOut.addVariables(predVars);
- origLiveOut.addVariables(_gen);
-
- _liveOut = new VariableSet();
- for (String name : candidateLO.getVariableNames()){
- if (origLiveOut.containsVariable(name)){
- _liveOut.addVariable(name, candidateLO.getVariable(name));
- }
- }
-
- initializebackwardLV(_liveOut);
-
- // set final warnSet: remove variables NOT in live out
- VariableSet finalWarnSet = new VariableSet();
- for (String varName : _warnSet.getVariableNames()){
- if (_liveOut.containsVariable(varName)){
- finalWarnSet.addVariable(varName,_warnSet.getVariable(varName));
- }
- }
- _warnSet = finalWarnSet;
-
- // for now just print the warn set
- for (String varName : _warnSet.getVariableNames()) {
- if( !ip.getIterVar().getName().equals( varName) )
- LOG.warn(_warnSet.getVariable(varName).printWarningLocation() + "Initialization of " + varName + " depends on for execution");
- }
-
- // Cannot remove kill variables
- _liveIn = new VariableSet();
- _liveIn.addVariables(_liveOut);
- _liveIn.addVariables(_gen);
-
- VariableSet liveInReturn = new VariableSet();
- liveInReturn.addVariables(_liveIn);
-
- return liveInReturn;
- }
-
-
- public void performConstantPropagation(HashMap<String, ConstIdentifier> currConstVars)
- throws LanguageException
- {
- IterablePredicate ip = getIterPredicate();
-
- // handle replacement in from expression
- Expression replacementExpr = replaceConstantVar(ip.getFromExpr(), currConstVars);
- if (replacementExpr != null)
- ip.setFromExpr(replacementExpr);
-
- // handle replacment in to expression
- replacementExpr = replaceConstantVar(ip.getToExpr(), currConstVars);
- if (replacementExpr != null)
- ip.setToExpr(replacementExpr);
-
- // handle replacement in increment expression
- replacementExpr = replaceConstantVar(ip.getIncrementExpr(), currConstVars);
- if (replacementExpr != null)
- ip.setIncrementExpr(replacementExpr);
- }
-
- private Expression replaceConstantVar(Expression expr, HashMap<String, ConstIdentifier> currConstVars)
- {
- Expression ret = null;
-
- if (expr instanceof DataIdentifier && !(expr instanceof IndexedIdentifier))
- {
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)expr).getName();
- if (currConstVars.containsKey(identifierName))
- {
- ConstIdentifier constValue = currConstVars.get(identifierName);
- //AUTO CASTING (using runtime operations for consistency)
- switch( constValue.getValueType() )
- {
- case DOUBLE:
- ret = new IntIdentifier(new DoubleObject(((DoubleIdentifier)constValue).getValue()).getLongValue(),
- expr.getFilename(), expr.getBeginLine(), expr.getBeginColumn(),
- expr.getEndLine(), expr.getEndColumn());
- break;
- case INT:
- ret = new IntIdentifier((IntIdentifier)constValue,
- expr.getFilename(), expr.getBeginLine(), expr.getBeginColumn(),
- expr.getEndLine(), expr.getEndColumn());
- break;
- case BOOLEAN:
- ret = new IntIdentifier(new BooleanObject(((BooleanIdentifier)constValue).getValue()).getLongValue(),
- expr.getFilename(), expr.getBeginLine(), expr.getBeginColumn(),
- expr.getEndLine(), expr.getEndColumn());
- break;
-
- default:
- //do nothing
- }
- }
- }
- else
- {
- //do nothing, cannot replace full expression
- ret = expr;
- }
-
- return ret;
- }
-
- /////////
- // materialized hops recompilation flags
- ////
-
- public void updatePredicateRecompilationFlags()
- throws HopsException
- {
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
- {
- _requiresFromRecompile = Recompiler.requiresRecompilation(getFromHops());
- _requiresToRecompile = Recompiler.requiresRecompilation(getToHops());
- _requiresIncrementRecompile = Recompiler.requiresRecompilation(getIncrementHops());
- }
- }
-
- public boolean requiresFromRecompilation()
- {
- return _requiresFromRecompile;
- }
-
- public boolean requiresToRecompilation()
- {
- return _requiresToRecompile;
- }
-
- public boolean requiresIncrementRecompilation()
- {
- return _requiresIncrementRecompile;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/FunctionCallIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/FunctionCallIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/FunctionCallIdentifier.java
deleted file mode 100644
index 2e67f01..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/FunctionCallIdentifier.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.io.IOException;
-
-
-public class FunctionCallIdentifier extends DataIdentifier
-{
-
- private ArrayList<ParameterExpression> _paramExprs;
- private FunctCallOp _opcode; // stores whether internal or external
- private String _namespace; // namespace of the function being called (null if current namespace is to be used)
-
- /**
- * setFunctionName: sets the function namespace (if specified) and name
- * @param functionName the (optional) namespace information and name of function. If both namespace and name are specified, they are concatinated with "::"
- * @throws ParseException
- */
- public void setFunctionName(String functionName) throws ParseException{
- _name = functionName;
- }
-
- public void setFunctionNamespace(String passed) throws ParseException{
- _namespace = passed;
- }
-
- public String getNamespace(){
- return _namespace;
- }
-
- public ArrayList<ParameterExpression> getParamExprs(){
- return _paramExprs;
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- ArrayList<ParameterExpression> newParameterExpressions = new ArrayList<ParameterExpression>();
- for (ParameterExpression paramExpr : _paramExprs)
- newParameterExpressions.add(new ParameterExpression(paramExpr.getName(), paramExpr.getExpr().rewriteExpression(prefix)));
-
- // rewrite each output expression
- FunctionCallIdentifier fci = new FunctionCallIdentifier(newParameterExpressions);
-
- fci.setBeginLine(this.getBeginLine());
- fci.setBeginColumn(this.getBeginColumn());
- fci.setEndLine(this.getEndLine());
- fci.setEndColumn(this.getEndColumn());
-
- fci._name = this._name;
- fci._namespace = this._namespace;
- fci._opcode = this._opcode;
- fci._kind = Kind.FunctionCallOp;
-
- return fci;
- }
-
-
-
- public FunctionCallIdentifier(){}
-
- public FunctionCallIdentifier(ArrayList<ParameterExpression> paramExpressions) {
-
- _paramExprs = paramExpressions;
- _opcode = null;
- _kind = Kind.FunctionCallOp;
- }
-
-
-
- public FunctCallOp getOpCode() {
- return _opcode;
- }
-
- /**
- * Validate parse tree : Process ExtBuiltinFunction Expression is an
- * assignment statement
- *
- * NOTE: this does not override the normal validateExpression because it needs to pass dmlp!
- *
- * @throws LanguageException
- */
- public void validateExpression(DMLProgram dmlp, HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, IOException
- {
- // Step 1: check the namespace exists, and that function is defined in the namespace
- if (dmlp.getNamespaces().get(_namespace) == null){
- raiseValidateError("namespace " + _namespace + " is not defined ", conditional);
- }
- FunctionStatementBlock fblock = dmlp.getFunctionStatementBlock(_namespace, _name);
- if (fblock == null){
- raiseValidateError("function " + _name + " is undefined in namespace " + _namespace, conditional);
- }
-
- // Step 2: set opcode (whether internal or external function) -- based on whether FunctionStatement
- // in FunctionStatementBlock is ExternalFunctionStatement or FunctionStatement
- if (fblock.getStatement(0) instanceof ExternalFunctionStatement)
- _opcode = Expression.FunctCallOp.EXTERNAL;
- else
- _opcode = Expression.FunctCallOp.INTERNAL;
-
- // Step 3: check all parameters to be either unnamed or named for functions
- boolean hasNamed = false, hasUnnamed = false;
- for( ParameterExpression paramExpr : _paramExprs ) {
- if (paramExpr.getName() == null)
- hasUnnamed = true;
- else
- hasNamed = true;
- }
- if (hasNamed && hasUnnamed){
- raiseValidateError(" In DML, functions can only have named parameters " +
- "(e.g., name1=value1, name2=value2) or unnamed parameters (e.g, value1, value2). " +
- _name + " has both parameter types.", conditional);
- }
-
- // Step 4: validate expressions for each passed parameter
- for( ParameterExpression paramExpr : _paramExprs ) {
- if (paramExpr.getExpr() instanceof FunctionCallIdentifier) {
- raiseValidateError("UDF function call not supported as parameter to function call", false);
- }
- paramExpr.getExpr().validateExpression(ids, constVars, conditional);
- }
-
- // Step 5: constant propagation into function call statement
- for( ParameterExpression paramExpr : _paramExprs ) {
- Expression expri = paramExpr.getExpr();
- if( expri instanceof DataIdentifier && !(expri instanceof IndexedIdentifier)
- && constVars.containsKey(((DataIdentifier)expri).getName()) )
- {
- //replace varname with constant in function call expression
- paramExpr.setExpr(constVars.get(((DataIdentifier)expri).getName()));
- }
- }
-
- // Step 6: check correctness of number of arguments and their types
- FunctionStatement fstmt = (FunctionStatement)fblock.getStatement(0);
- if (fstmt.getInputParams().size() < _paramExprs.size()) {
- raiseValidateError("function " + _name
- + " has incorrect number of parameters. Function requires "
- + fstmt.getInputParams().size() + " but was called with " + _paramExprs.size(), conditional);
- }
-
- // Step 7: set the outputs for the function
- _outputs = new Identifier[fstmt.getOutputParams().size()];
- for(int i=0; i < fstmt.getOutputParams().size(); i++) {
- _outputs[i] = new DataIdentifier(fstmt.getOutputParams().get(i));
- }
-
- return;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- if (_namespace != null && _namespace.length() > 0 && !_namespace.equals(".defaultNS"))
- sb.append(_namespace + "::");
- sb.append(_name);
- sb.append(" ( ");
-
- for (int i = 0; i < _paramExprs.size(); i++){
- sb.append(_paramExprs.get(i).toString());
- if (i<_paramExprs.size() - 1)
- sb.append(",");
- }
- sb.append(" )");
- return sb.toString();
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- for (int i = 0; i < _paramExprs.size(); i++)
- result.addVariables(_paramExprs.get(i).getExpr().variablesRead());
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- for (int i=0; i< _outputs.length; i++)
- result.addVariable( ((DataIdentifier)_outputs[i]).getName(), (DataIdentifier)_outputs[i] );
- return result;
- }
-
- @Override
- public boolean multipleReturns() {
- return true;
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/FunctionStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/FunctionStatement.java b/src/main/java/com/ibm/bi/dml/parser/FunctionStatement.java
deleted file mode 100644
index 98d58d0..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/FunctionStatement.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.Lop;
-
-
-public class FunctionStatement extends Statement
-{
-
- private ArrayList<StatementBlock> _body;
- protected String _name;
- protected ArrayList <DataIdentifier> _inputParams;
- protected ArrayList <DataIdentifier> _outputParams;
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should not call rewriteStatement for FunctionStatement");
- throw new LanguageException(this.printErrorLocation() + "should not call rewriteStatement for FunctionStatement");
- }
-
- public FunctionStatement(){
- _body = new ArrayList<StatementBlock>();
- _name = null;
- _inputParams = new ArrayList<DataIdentifier>();
- _outputParams = new ArrayList<DataIdentifier>();
- }
-
- public ArrayList<DataIdentifier> getInputParams(){
- return _inputParams;
- }
-
- public ArrayList<DataIdentifier> getOutputParams(){
- return _outputParams;
- }
-
- public void setInputParams(ArrayList<DataIdentifier> inputParams){
- _inputParams = inputParams;
- }
-
- public void setOutputParams(ArrayList<DataIdentifier> outputParams){
- _outputParams = outputParams;
- }
-
- public void setName(String fname){
- _name = fname;
- }
-
- public String getName(){
- return _name;
- }
-
- public void addStatementBlock(StatementBlock sb){
- _body.add(sb);
- }
-
- public ArrayList<StatementBlock> getBody(){
- return _body;
- }
-
- public void setBody(ArrayList<StatementBlock> body){
- _body = body;
- }
-
-
- @Override
- public boolean controlStatement() {
- return true;
- }
-
- public void mergeStatementBlocks(){
- _body = StatementBlock.mergeStatementBlocks(_body);
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(_name + " = ");
-
- sb.append("function ( ");
-
- for (int i=0; i<_inputParams.size(); i++){
- DataIdentifier curr = _inputParams.get(i);
- sb.append(curr.getName());
- if (curr.getDefaultValue() != null) sb.append(" = " + curr.getDefaultValue());
- if (i < _inputParams.size()-1) sb.append(", ");
- }
- sb.append(") return (");
-
- for (int i=0; i<_outputParams.size(); i++){
- sb.append(_outputParams.get(i).getName());
- if (i < _outputParams.size()-1) sb.append(", ");
- }
- sb.append(") { \n");
-
- for (StatementBlock block : _body){
- sb.append(block.toString());
- }
- sb.append("} \n");
- return sb.toString();
- }
-
- public void initializeforwardLV(VariableSet activeIn) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for FunctionStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for FunctionStatement");
- }
-
- public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for FunctionStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for FunctionStatement");
-
- }
-
- @Override
- public VariableSet variablesRead() {
- LOG.warn(this.printWarningLocation() + " -- should not call variablesRead from FunctionStatement ");
- return new VariableSet();
- }
-
- @Override
- public VariableSet variablesUpdated() {
- LOG.warn(this.printWarningLocation() + " -- should not call variablesRead from FunctionStatement ");
- return new VariableSet();
- }
-
- public static String[] createFunctionCallVariables( ArrayList<Lop> lops )
- {
- String[] ret = new String[lops.size()]; //vars in order
-
- for( int i=0; i<lops.size(); i++ )
- {
- Lop llops = lops.get(i);
- if( llops.getType()==Lop.Type.Data )
- ret[i] = llops.getOutputParameters().getLabel();
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/FunctionStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/FunctionStatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/FunctionStatementBlock.java
deleted file mode 100644
index faf9ac5..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/FunctionStatementBlock.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class FunctionStatementBlock extends StatementBlock
-{
-
- private boolean _recompileOnce = false;
-
- /**
- * TODO: DRB: This needs to be changed to reflect:
- *
- * 1) Default values for variables -- need to add R styled check here to make sure that once vars with
- * default values start, they keep going to the right
- *
- * 2) The other parameters for External Functions
- * @throws IOException
- */
- @Override
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
-
- if (_statements.size() > 1){
- LOG.error(this.printBlockErrorLocation() + "FunctionStatementBlock should have only 1 statement (FunctionStatement)");
- throw new LanguageException(this.printBlockErrorLocation() + "FunctionStatementBlock should have only 1 statement (FunctionStatement)");
- }
- FunctionStatement fstmt = (FunctionStatement) _statements.get(0);
-
- // validate all function input parameters
- ArrayList<DataIdentifier> inputValues = fstmt.getInputParams();
- for( DataIdentifier inputValue : inputValues ) {
- //check all input matrices have value type double
- if( inputValue.getDataType()==DataType.MATRIX && inputValue.getValueType()!=ValueType.DOUBLE ) {
- raiseValidateError("for function " + fstmt.getName() + ", input variable " + inputValue.getName()
- + " has an unsupported value type of " + inputValue.getValueType() + ".", false);
- }
- }
-
- // handle DML-bodied functions
- if (!(fstmt instanceof ExternalFunctionStatement))
- {
- // perform validate for function body
- this._dmlProg = dmlProg;
- for(StatementBlock sb : fstmt.getBody())
- {
- ids = sb.validate(dmlProg, ids, constVars, conditional);
- constVars = sb.getConstOut();
- }
- if (fstmt.getBody().size() > 0)
- _constVarsIn.putAll(fstmt.getBody().get(0).getConstIn());
-
- if (fstmt.getBody().size() > 1)
- _constVarsOut.putAll(fstmt.getBody().get(fstmt.getBody().size()-1).getConstOut());
-
- // for each return value, check variable is defined and validate the return type
- // if returnValue type known incorrect, then throw exception
- ArrayList<DataIdentifier> returnValues = fstmt.getOutputParams();
- for (DataIdentifier returnValue : returnValues){
- DataIdentifier curr = ids.getVariable(returnValue.getName());
- if (curr == null){
- raiseValidateError("for function " + fstmt.getName() + ", return variable " + returnValue.getName() + " must be defined in function ", conditional);
- }
-
- if (curr.getDataType() == DataType.UNKNOWN){
- raiseValidateError("for function " + fstmt.getName() + ", return variable " + curr.getName() + " data type of " + curr.getDataType() + " may not match data type in function signature of " + returnValue.getDataType(), true);
- }
-
- if (curr.getValueType() == ValueType.UNKNOWN){
- raiseValidateError("for function " + fstmt.getName() + ", return variable " + curr.getName() + " data type of " + curr.getValueType() + " may not match data type in function signature of " + returnValue.getValueType(), true);
- }
-
- if (curr.getDataType() != DataType.UNKNOWN && !curr.getDataType().equals(returnValue.getDataType()) ){
- raiseValidateError("for function " + fstmt.getName() + ", return variable " + curr.getName() + " data type of " + curr.getDataType() + " does not match data type in function signature of " + returnValue.getDataType(), conditional);
- }
-
- if (curr.getValueType() != ValueType.UNKNOWN && !curr.getValueType().equals(returnValue.getValueType())){
-
- // attempt to convert value type: handle conversion from scalar DOUBLE or INT
- if (curr.getDataType() == DataType.SCALAR && returnValue.getDataType() == DataType.SCALAR){
- if (returnValue.getValueType() == ValueType.DOUBLE){
- if (curr.getValueType() == ValueType.INT){
- IntIdentifier currIntValue = (IntIdentifier)constVars.get(curr.getName());
- if (currIntValue != null){
- DoubleIdentifier currDoubleValue = new DoubleIdentifier(currIntValue.getValue(),
- curr.getFilename(), curr.getBeginLine(), curr.getBeginColumn(),
- curr.getEndLine(), curr.getEndColumn());
- constVars.put(curr.getName(), currDoubleValue);
- }
- LOG.warn(curr.printWarningLocation() + "for function " + fstmt.getName()
- + ", return variable " + curr.getName() + " value type of "
- + curr.getValueType() + " does not match value type in function signature of "
- + returnValue.getValueType() + " but was safely cast");
- curr.setValueType(ValueType.DOUBLE);
- ids.addVariable(curr.getName(), curr);
- }
- else {
- // THROW EXCEPTION -- CANNOT CONVERT
- LOG.error(curr.printErrorLocation() + "for function " + fstmt.getName()
- + ", return variable " + curr.getName() + " value type of "
- + curr.getValueType() + " does not match value type in function signature of "
- + returnValue.getValueType() + " and cannot safely cast value");
- throw new LanguageException(curr.printErrorLocation() + "for function "
- + fstmt.getName() + ", return variable " + curr.getName()
- + " value type of " + curr.getValueType()
- + " does not match value type in function signature of "
- + returnValue.getValueType() + " and cannot safely cast value");
- }
- }
- if (returnValue.getValueType() == ValueType.INT){
- // THROW EXCEPTION -- CANNOT CONVERT
- LOG.error(curr.printErrorLocation() + "for function " + fstmt.getName()
- + ", return variable " + curr.getName() + " value type of "
- + curr.getValueType() + " does not match value type in function signature of "
- + returnValue.getValueType() + " and cannot safely cast " + curr.getValueType()
- + " as " + returnValue.getValueType());
- throw new LanguageException(curr.printErrorLocation() + "for function " + fstmt.getName()
- + ", return variable " + curr.getName() + " value type of " + curr.getValueType()
- + " does not match value type in function signature of "
- + returnValue.getValueType() + " and cannot safely cast " + curr.getValueType()
- + " as " + returnValue.getValueType());
-
- }
- }
- else {
- LOG.error(curr.printErrorLocation() + "for function " + fstmt.getName() + ", return variable " + curr.getName() + " value type of " + curr.getValueType() + " does not match value type in function signature of " + returnValue.getValueType() + " and cannot safely cast double as int");
- throw new LanguageException(curr.printErrorLocation() + "for function " + fstmt.getName() + ", return variable " + curr.getName() + " value type of " + curr.getValueType() + " does not match value type in function signature of " + returnValue.getValueType() + " and cannot safely cast " + curr.getValueType() + " as " + returnValue.getValueType());
- }
- }
-
- }
- }
- // handle external functions
- else
- {
- //validate specified attributes and attribute values
- ExternalFunctionStatement efstmt = (ExternalFunctionStatement) fstmt;
- efstmt.validateParameters(this);
-
- //validate child statements
- this._dmlProg = dmlProg;
- for(StatementBlock sb : efstmt.getBody())
- {
- ids = sb.validate(dmlProg, ids, constVars, conditional);
- constVars = sb.getConstOut();
- }
- }
-
-
-
- return ids;
- }
-
- public FunctionType getFunctionOpType()
- {
- FunctionType ret = FunctionType.UNKNOWN;
-
- FunctionStatement fstmt = (FunctionStatement) _statements.get(0);
- if (fstmt instanceof ExternalFunctionStatement)
- {
- ExternalFunctionStatement efstmt = (ExternalFunctionStatement) fstmt;
- String execType = efstmt.getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
- if( execType!=null ){
- if(execType.equals(ExternalFunctionStatement.IN_MEMORY))
- ret = FunctionType.EXTERNAL_MEM;
- else
- ret = FunctionType.EXTERNAL_FILE;
- }
- }
- else
- {
- ret = FunctionType.DML;
- }
-
- return ret;
- }
-
- public VariableSet initializeforwardLV(VariableSet activeInPassed) throws LanguageException {
-
- FunctionStatement fstmt = (FunctionStatement)_statements.get(0);
- if (_statements.size() > 1){
- LOG.error(this.printBlockErrorLocation() + "FunctionStatementBlock should have only 1 statement (while statement)");
- throw new LanguageException(this.printBlockErrorLocation() + "FunctionStatementBlock should have only 1 statement (while statement)");
- }
-
- _read = new VariableSet();
- _gen = new VariableSet();
-
- VariableSet current = new VariableSet();
- current.addVariables(activeInPassed);
-
- for( StatementBlock sb : fstmt.getBody() )
- {
- current = sb.initializeforwardLV(current);
-
- // for each generated variable in this block, check variable not killed
- // in prior statement block in while stmt blody
- for (String varName : sb._gen.getVariableNames()){
-
- // IF the variable is NOT set in the while loop PRIOR to this stmt block,
- // THEN needs to be generated
- if (!_kill.getVariableNames().contains(varName)){
- _gen.addVariable(varName, sb._gen.getVariable(varName));
- }
- }
-
- _read.addVariables(sb._read);
- _updated.addVariables(sb._updated);
-
- // only add kill variables for statement blocks guaranteed to execute
- if (!(sb instanceof WhileStatementBlock) && !(sb instanceof ForStatementBlock) ){
- _kill.addVariables(sb._kill);
- }
- }
-
- // activeOut includes variables from passed live in and updated in the while body
- _liveOut = new VariableSet();
- _liveOut.addVariables(current);
- _liveOut.addVariables(_updated);
- return _liveOut;
- }
-
- public VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException{
-
- FunctionStatement wstmt = (FunctionStatement)_statements.get(0);
-
- VariableSet lo = new VariableSet();
- lo.addVariables(loPassed);
-
- // calls analyze for each statement block in while stmt body
- int numBlocks = wstmt.getBody().size();
- for (int i = numBlocks - 1; i >= 0; i--){
- lo = wstmt.getBody().get(i).analyze(lo);
- }
-
- VariableSet loReturn = new VariableSet();
- loReturn.addVariables(lo);
- return loReturn;
-
- }
-
-
- public ArrayList<Hop> get_hops() throws HopsException {
-
- if (_hops != null && _hops.size() > 0){
- LOG.error(this.printBlockErrorLocation() + "there should be no HOPs associated with the FunctionStatementBlock");
- throw new HopsException(this.printBlockErrorLocation() + "there should be no HOPs associated with the FunctionStatementBlock");
- }
-
- return _hops;
- }
-
-
- public VariableSet analyze(VariableSet loPassed) throws LanguageException{
- LOG.error(this.printBlockErrorLocation() + "Both liveIn and liveOut variables need to be specified for liveness analysis for FunctionStatementBlock");
- throw new LanguageException(this.printBlockErrorLocation() + "Both liveIn and liveOut variables need to be specified for liveness analysis for FunctionStatementBlock");
- }
-
-
- public VariableSet analyze(VariableSet liPassed, VariableSet loPassed) throws LanguageException{
-
- VariableSet candidateLO = new VariableSet();
- candidateLO.addVariables(loPassed);
- candidateLO.addVariables(_gen);
-
- VariableSet origLiveOut = new VariableSet();
- origLiveOut.addVariables(_liveOut);
-
- _liveOut = new VariableSet();
- for (String name : candidateLO.getVariableNames()){
- if (origLiveOut.containsVariable(name)){
- _liveOut.addVariable(name, candidateLO.getVariable(name));
- }
- }
-
- initializebackwardLV(_liveOut);
-
- // Cannot remove kill variables
- _liveIn = new VariableSet();
- _liveIn.addVariables(liPassed);
-
- VariableSet liveInReturn = new VariableSet();
- liveInReturn.addVariables(_liveIn);
-
- return liveInReturn;
- }
-
- public void setRecompileOnce( boolean flag ) {
- _recompileOnce = flag;
- }
-
- public boolean isRecompileOnce() {
- return _recompileOnce;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/Identifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/Identifier.java b/src/main/java/com/ibm/bi/dml/parser/Identifier.java
deleted file mode 100644
index 522b513..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/Identifier.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-
-public abstract class Identifier extends Expression
-{
-
- protected DataType _dataType;
- protected ValueType _valueType;
- protected long _dim1;
- protected long _dim2;
- protected long _rows_in_block;
- protected long _columns_in_block;
- protected long _nnz;
- protected FormatType _formatType;
-
- public Identifier(Identifier i)
- {
- _dataType = i.getDataType();
- _valueType = i.getValueType();
- if( i instanceof IndexedIdentifier ) {
- IndexedIdentifier ixi = (IndexedIdentifier)i;
- _dim1 = ixi.getOrigDim1();
- _dim2 = ixi.getOrigDim2();
- }
- else {
- _dim1 = i.getDim1();
- _dim2 = i.getDim2();
- }
- _rows_in_block = i.getRowsInBlock();
- _columns_in_block = i.getColumnsInBlock();
- _nnz = i.getNnz();
- _formatType = i.getFormatType();
-
- // copy position information
- setBeginLine(i.getBeginLine());
- setBeginColumn(i.getBeginColumn());
- setEndLine(i.getEndLine());
- setEndColumn(i.getEndColumn());
- }
-
- public Identifier()
- {
- _dim1 = -1;
- _dim2 = -1;
- _dataType = DataType.UNKNOWN;
- _valueType = ValueType.UNKNOWN;
- _rows_in_block = -1;
- _columns_in_block = -1;
- _nnz = -1;
- setOutput(this);
- _formatType = null;
- }
-
- public void setProperties(Identifier i)
- {
- if (i == null)
- return;
-
- _dataType = i.getDataType();
- _valueType = i.getValueType();
- if (i instanceof IndexedIdentifier) {
- _dim1 = ((IndexedIdentifier)i).getOrigDim1();
- _dim2 = ((IndexedIdentifier)i).getOrigDim2();
- }
- else {
- _dim1 = i.getDim1();
- _dim2 = i.getDim2();
- }
- _rows_in_block = i.getRowsInBlock();
- _columns_in_block = i.getColumnsInBlock();
- _nnz = i.getNnz();
- _formatType = i.getFormatType();
-
- }
-
- public void setDimensionValueProperties(Identifier i)
- {
- if (i instanceof IndexedIdentifier) {
- IndexedIdentifier ixi = (IndexedIdentifier)i;
- _dim1 = ixi.getOrigDim1();
- _dim2 = ixi.getOrigDim2();
- }
- else {
- _dim1 = i.getDim1();
- _dim2 = i.getDim2();
- }
- _nnz = i.getNnz();
- _dataType = i.getDataType();
- _valueType = i.getValueType();
- }
-
- public void setDataType(DataType dt){
- _dataType = dt;
- }
-
- public void setValueType(ValueType vt){
- _valueType = vt;
- }
-
- public void setFormatType(FormatType ft){
- _formatType = ft;
- }
-
- public void setDimensions(long dim1, long dim2){
- _dim1 = dim1;
- _dim2 = dim2;
- }
-
- public void setBlockDimensions(long dim1, long dim2){
- _rows_in_block = dim1;
- _columns_in_block = dim2;
- }
-
- public void setNnz(long nnzs){
- _nnz = nnzs;
- }
-
- public long getDim1(){
- return _dim1;
- }
-
- public long getDim2(){
- return _dim2;
- }
-
- public DataType getDataType(){
- return _dataType;
- }
-
- public ValueType getValueType(){
- return _valueType;
- }
-
- public FormatType getFormatType(){
- return _formatType;
- }
-
- public long getRowsInBlock(){
- return _rows_in_block;
- }
-
- public long getColumnsInBlock(){
- return _columns_in_block;
- }
-
- public long getNnz(){
- return _nnz;
- }
-
- @Override
- public void validateExpression(HashMap<String,DataIdentifier> ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
-
- if( getOutput() instanceof DataIdentifier ) {
-
- // set properties for Data identifier
- String name = ((DataIdentifier)this.getOutput()).getName();
- Identifier id = ids.get(name);
- if ( id == null ){
- //undefined variables are always treated unconditionally as error in order to prevent common script-level bugs
- raiseValidateError("Undefined Variable (" + name + ") used in statement", false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- this.getOutput().setProperties(id);
-
- // validate IndexedIdentifier -- which is substype of DataIdentifer with index
- if (this.getOutput() instanceof IndexedIdentifier){
-
- // validate the row / col index bounds (if defined)
- IndexedIdentifier indexedIdentiferOut = (IndexedIdentifier)this.getOutput();
-
- if (indexedIdentiferOut.getRowLowerBound() != null) {
- indexedIdentiferOut.getRowLowerBound().validateExpression(ids, constVars, conditional);
-
- Expression tempExpr = indexedIdentiferOut.getRowLowerBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
- raiseValidateError("Matrix values for row lower index bound are not supported, which includes indexed identifiers.", conditional);
- }
-
- }
- if (indexedIdentiferOut.getRowUpperBound() != null) {
- indexedIdentiferOut.getRowUpperBound().validateExpression(ids, constVars, conditional);
-
- Expression tempExpr = indexedIdentiferOut.getRowUpperBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
- raiseValidateError("Matrix values for row upper index bound are not supported, which includes indexed identifiers.", conditional);
- }
-
- }
- if (indexedIdentiferOut.getColLowerBound() != null) {
- indexedIdentiferOut.getColLowerBound().validateExpression(ids,constVars, conditional);
-
- Expression tempExpr = indexedIdentiferOut.getColLowerBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
- raiseValidateError("Matrix values for column lower index bound are not supported, which includes indexed identifiers.", conditional);
- }
-
- }
- if (indexedIdentiferOut.getColUpperBound() != null) {
- indexedIdentiferOut.getColUpperBound().validateExpression(ids, constVars, conditional);
-
- Expression tempExpr = indexedIdentiferOut.getColUpperBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
- raiseValidateError("Matrix values for column upper index bound are not supported, which includes indexed identifiers.", conditional);
- }
-
- }
-
- IndexPair updatedIndices = ((IndexedIdentifier)this.getOutput()).calculateIndexedDimensions(ids, constVars, conditional);
- ((IndexedIdentifier)this.getOutput()).setDimensions(updatedIndices._row, updatedIndices._col);
-
- }
-
- } else {
- this.getOutput().setProperties(this.getOutput());
- }
- }
-
- public void computeDataType() {
-
- if ((_dim1 == 0) && (_dim2 == 0)) {
- _dataType = DataType.SCALAR;
- } else if ((_dim1 >= 1) || (_dim2 >= 1)){
- // Vector also set as matrix
- // Data type is set as matrix, if either of dimensions is -1
- _dataType = DataType.MATRIX;
- } else _dataType = DataType.UNKNOWN;
-
- }
-
- public void setBooleanProperties(){
- _dataType = DataType.SCALAR;
- _valueType = ValueType.BOOLEAN;
- _dim1 = 0;
- _dim2 = 0;
- _rows_in_block = 0;
- _columns_in_block = 0;
- _nnz = -1;
- _formatType = null;
- }
-
- public void setIntProperties(){
- _dataType = DataType.SCALAR;
- _valueType = ValueType.INT;
- _dim1 = 0;
- _dim2 = 0;
- _rows_in_block = 0;
- _columns_in_block = 0;
- _nnz = -1;
- _formatType = null;
- }
-
-
- public boolean isScalarBoolean(){
- return (_valueType == ValueType.BOOLEAN) && (_dataType == DataType.SCALAR);
- }
-
- public boolean dimsKnown(){
- return ( _dim1 > 0 && _dim2 > 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/IfStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/IfStatement.java b/src/main/java/com/ibm/bi/dml/parser/IfStatement.java
deleted file mode 100644
index 620d9de..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/IfStatement.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-
-
-public class IfStatement extends Statement
-{
-
- private ConditionalPredicate _predicate;
- private ArrayList<StatementBlock> _ifBody;
- private ArrayList<StatementBlock> _elseBody;
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should not call rewriteStatement for IfStatement");
- throw new LanguageException(this.printErrorLocation() + "should not call rewriteStatement for IfStatement");
- }
-
- public IfStatement(){
- _predicate = null;
- _ifBody = new ArrayList<StatementBlock>();
- _elseBody = new ArrayList<StatementBlock>();
- }
-
- public void setConditionalPredicate(ConditionalPredicate pred){
- _predicate = pred;
- }
-
-
- public void addStatementBlockIfBody(StatementBlock sb){
- _ifBody.add(sb);
- }
-
- public void addStatementBlockElseBody(StatementBlock sb){
- _elseBody.add(sb);
- }
-
- public ConditionalPredicate getConditionalPredicate(){
- return _predicate;
- }
-
- public ArrayList<StatementBlock> getIfBody(){
- return _ifBody;
- }
-
- public ArrayList<StatementBlock> getElseBody(){
- return _elseBody;
- }
-
- public void setIfBody(ArrayList<StatementBlock> body){
- _ifBody = body;
- }
-
- public void setElseBody(ArrayList<StatementBlock> body){
- _elseBody = body;
- }
-
-
- @Override
- public boolean controlStatement() {
- return true;
- }
-
- public void initializeforwardLV(VariableSet activeIn) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for IfStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for IfStatement");
-
- }
-
- public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{
- LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for IfStatement");
- throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for IfStatement");
-
- }
-
- public void mergeStatementBlocksIfBody(){
- _ifBody = StatementBlock.mergeStatementBlocks(_ifBody);
- }
-
- public void mergeStatementBlocksElseBody(){
- if (!_elseBody.isEmpty())
- _elseBody = StatementBlock.mergeStatementBlocks(_elseBody);
- }
-
- public VariableSet variablesReadIfBody() {
-
- return null;
-
- }
-
- public VariableSet variablesReadElseBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesReadElseBody from IfStatement ");
- return null;
- }
-
- public VariableSet variablesUpdatedIfBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesUpdatedIfBody from IfStatement ");
- return null;
- }
-
- public VariableSet variablesUpdatedElseBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesUpdatedElseBody from IfStatement ");
- return null;
- }
-
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append("if ( ");
- sb.append(_predicate.toString());
- sb.append(") { \n");
- for (StatementBlock block : _ifBody){
- sb.append(block.toString());
- }
- sb.append("}\n");
- if (!_elseBody.isEmpty()){
- sb.append(" else { \n");
- for (StatementBlock block : _elseBody){
- sb.append(block.toString());
- }
- sb.append("}\n");
- }
- return sb.toString();
- }
-
-
- public VariableSet variablesKill() {
- return new VariableSet();
- }
-
- @Override
- public VariableSet variablesRead() {
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesRead from IfStatement ");
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesUpdated from IfStatement ");
- return null;
- }
-}
[74/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CumulativeSplitAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CumulativeSplitAggregate.java b/src/main/java/org/apache/sysml/lops/CumulativeSplitAggregate.java
index fc22c5e..9c3fb94 100644
--- a/src/main/java/org/apache/sysml/lops/CumulativeSplitAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/CumulativeSplitAggregate.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Data.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Data.java b/src/main/java/org/apache/sysml/lops/Data.java
index eca031d..ff8f1e7 100644
--- a/src/main/java/org/apache/sysml/lops/Data.java
+++ b/src/main/java/org/apache/sysml/lops/Data.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.OutputParameters.Format;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
@@ -140,7 +140,7 @@ public class Data extends Lop
// If the input is data transform, then csv write can be piggybacked onto TRANSFORM job.
// Otherwise, the input must be converted to csv format via WriteCSV MR job.
if ( input instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)input).getOp() == com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
+ && ((ParameterizedBuiltin)input).getOp() == org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
lps.addCompatibility(JobType.TRANSFORM);
definesMRJob = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/DataGen.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/DataGen.java b/src/main/java/org/apache/sysml/lops/DataGen.java
index 9cea440..45df302 100644
--- a/src/main/java/org/apache/sysml/lops/DataGen.java
+++ b/src/main/java/org/apache/sysml/lops/DataGen.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.OutputParameters.Format;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/DataPartition.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/DataPartition.java b/src/main/java/org/apache/sysml/lops/DataPartition.java
index 1434611..c06cc5c 100644
--- a/src/main/java/org/apache/sysml/lops/DataPartition.java
+++ b/src/main/java/org/apache/sysml/lops/DataPartition.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/FunctionCallCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/FunctionCallCP.java b/src/main/java/org/apache/sysml/lops/FunctionCallCP.java
index e5ee9be..12ea167 100644
--- a/src/main/java/org/apache/sysml/lops/FunctionCallCP.java
+++ b/src/main/java/org/apache/sysml/lops/FunctionCallCP.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Group.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Group.java b/src/main/java/org/apache/sysml/lops/Group.java
index 7af0ca1..2555748 100644
--- a/src/main/java/org/apache/sysml/lops/Group.java
+++ b/src/main/java/org/apache/sysml/lops/Group.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
* Lop to represent a grouping operation.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/GroupedAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/GroupedAggregate.java b/src/main/java/org/apache/sysml/lops/GroupedAggregate.java
index 974844b..ca06bdb 100644
--- a/src/main/java/org/apache/sysml/lops/GroupedAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/GroupedAggregate.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/LeftIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LeftIndex.java b/src/main/java/org/apache/sysml/lops/LeftIndex.java
index d81b250..46c2f68 100644
--- a/src/main/java/org/apache/sysml/lops/LeftIndex.java
+++ b/src/main/java/org/apache/sysml/lops/LeftIndex.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class LeftIndex extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index 751ca1f..45a8810 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/LopProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LopProperties.java b/src/main/java/org/apache/sysml/lops/LopProperties.java
index 3268e1d..75ac508 100644
--- a/src/main/java/org/apache/sysml/lops/LopProperties.java
+++ b/src/main/java/org/apache/sysml/lops/LopProperties.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
public class LopProperties
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/LopsException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LopsException.java b/src/main/java/org/apache/sysml/lops/LopsException.java
index 762a3e7..af0ed84 100644
--- a/src/main/java/org/apache/sysml/lops/LopsException.java
+++ b/src/main/java/org/apache/sysml/lops/LopsException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
public class LopsException extends DMLException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MMCJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MMCJ.java b/src/main/java/org/apache/sysml/lops/MMCJ.java
index dc37667..66faba7 100644
--- a/src/main/java/org/apache/sysml/lops/MMCJ.java
+++ b/src/main/java/org/apache/sysml/lops/MMCJ.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MMRJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MMRJ.java b/src/main/java/org/apache/sysml/lops/MMRJ.java
index 8722526..6cbdf1f 100644
--- a/src/main/java/org/apache/sysml/lops/MMRJ.java
+++ b/src/main/java/org/apache/sysml/lops/MMRJ.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MMTSJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MMTSJ.java b/src/main/java/org/apache/sysml/lops/MMTSJ.java
index d1d3053..9e051ba 100644
--- a/src/main/java/org/apache/sysml/lops/MMTSJ.java
+++ b/src/main/java/org/apache/sysml/lops/MMTSJ.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MMZip.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MMZip.java b/src/main/java/org/apache/sysml/lops/MMZip.java
index b8875fb..0a8b246 100644
--- a/src/main/java/org/apache/sysml/lops/MMZip.java
+++ b/src/main/java/org/apache/sysml/lops/MMZip.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MapMult.java b/src/main/java/org/apache/sysml/lops/MapMult.java
index e3d4d2e..65727e2 100644
--- a/src/main/java/org/apache/sysml/lops/MapMult.java
+++ b/src/main/java/org/apache/sysml/lops/MapMult.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class MapMult extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/MapMultChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MapMultChain.java b/src/main/java/org/apache/sysml/lops/MapMultChain.java
index 8badbc0..b181d49 100644
--- a/src/main/java/org/apache/sysml/lops/MapMultChain.java
+++ b/src/main/java/org/apache/sysml/lops/MapMultChain.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class MapMultChain extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/OutputParameters.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/OutputParameters.java b/src/main/java/org/apache/sysml/lops/OutputParameters.java
index 3772f68..c992919 100644
--- a/src/main/java/org/apache/sysml/lops/OutputParameters.java
+++ b/src/main/java/org/apache/sysml/lops/OutputParameters.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.hops.HopsException;
+import org.apache.sysml.hops.HopsException;
/**
* class to maintain output parameters for a lop.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/PMMJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/PMMJ.java b/src/main/java/org/apache/sysml/lops/PMMJ.java
index c540256..49f561e 100644
--- a/src/main/java/org/apache/sysml/lops/PMMJ.java
+++ b/src/main/java/org/apache/sysml/lops/PMMJ.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/PMapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/PMapMult.java b/src/main/java/org/apache/sysml/lops/PMapMult.java
index cb12999..db81d63 100644
--- a/src/main/java/org/apache/sysml/lops/PMapMult.java
+++ b/src/main/java/org/apache/sysml/lops/PMapMult.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class PMapMult extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
index c67a92b..5facd87 100644
--- a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
+++ b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/PartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/PartialAggregate.java b/src/main/java/org/apache/sysml/lops/PartialAggregate.java
index 68f4b74..34d2ed1 100644
--- a/src/main/java/org/apache/sysml/lops/PartialAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/PartialAggregate.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/PickByCount.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/PickByCount.java b/src/main/java/org/apache/sysml/lops/PickByCount.java
index f0cca24..b3c6874 100644
--- a/src/main/java/org/apache/sysml/lops/PickByCount.java
+++ b/src/main/java/org/apache/sysml/lops/PickByCount.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class PickByCount extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java b/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
index a36c0d3..78a4340 100644
--- a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
+++ b/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class RangeBasedReIndex extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/ReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/ReBlock.java b/src/main/java/org/apache/sysml/lops/ReBlock.java
index a442fa6..300587c 100644
--- a/src/main/java/org/apache/sysml/lops/ReBlock.java
+++ b/src/main/java/org/apache/sysml/lops/ReBlock.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.OutputParameters.Format;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/RepMat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/RepMat.java b/src/main/java/org/apache/sysml/lops/RepMat.java
index b838539..fdbe7fb 100644
--- a/src/main/java/org/apache/sysml/lops/RepMat.java
+++ b/src/main/java/org/apache/sysml/lops/RepMat.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class RepMat extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/SortKeys.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/SortKeys.java b/src/main/java/org/apache/sysml/lops/SortKeys.java
index 39b0ea7..17031b5 100644
--- a/src/main/java/org/apache/sysml/lops/SortKeys.java
+++ b/src/main/java/org/apache/sysml/lops/SortKeys.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashSet;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class SortKeys extends Lop
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Ternary.java b/src/main/java/org/apache/sysml/lops/Ternary.java
index 2feeaac..e02c840 100644
--- a/src/main/java/org/apache/sysml/lops/Ternary.java
+++ b/src/main/java/org/apache/sysml/lops/Ternary.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/TernaryAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/TernaryAggregate.java b/src/main/java/org/apache/sysml/lops/TernaryAggregate.java
index 2687b98..8d60e07 100644
--- a/src/main/java/org/apache/sysml/lops/TernaryAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/TernaryAggregate.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Transform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Transform.java b/src/main/java/org/apache/sysml/lops/Transform.java
index b4e6d13..807fde8 100644
--- a/src/main/java/org/apache/sysml/lops/Transform.java
+++ b/src/main/java/org/apache/sysml/lops/Transform.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/UAggOuterChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/UAggOuterChain.java b/src/main/java/org/apache/sysml/lops/UAggOuterChain.java
index fd36c9b..66ad428 100644
--- a/src/main/java/org/apache/sysml/lops/UAggOuterChain.java
+++ b/src/main/java/org/apache/sysml/lops/UAggOuterChain.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Unary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Unary.java b/src/main/java/org/apache/sysml/lops/Unary.java
index b29e822..8519e58 100644
--- a/src/main/java/org/apache/sysml/lops/Unary.java
+++ b/src/main/java/org/apache/sysml/lops/Unary.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/UnaryCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/UnaryCP.java b/src/main/java/org/apache/sysml/lops/UnaryCP.java
index c92fbfc..bdb1401 100644
--- a/src/main/java/org/apache/sysml/lops/UnaryCP.java
+++ b/src/main/java/org/apache/sysml/lops/UnaryCP.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedCrossEntropy.java b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropy.java
index df131c9..e8aabfb 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedCrossEntropy.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropy.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropyR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedCrossEntropyR.java b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropyR.java
index 58f3cc7..cd94f49 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedCrossEntropyR.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedCrossEntropyR.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedDivMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedDivMM.java b/src/main/java/org/apache/sysml/lops/WeightedDivMM.java
index ab6a858..c88bab3 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedDivMM.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedDivMM.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedDivMMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedDivMMR.java b/src/main/java/org/apache/sysml/lops/WeightedDivMMR.java
index 8f8522f..89517b6 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedDivMMR.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedDivMMR.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedSigmoid.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedSigmoid.java b/src/main/java/org/apache/sysml/lops/WeightedSigmoid.java
index 310ae37..b069fa5 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedSigmoid.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedSigmoid.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedSigmoidR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedSigmoidR.java b/src/main/java/org/apache/sysml/lops/WeightedSigmoidR.java
index de99bfd..6f71886 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedSigmoidR.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedSigmoidR.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedSquaredLoss.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedSquaredLoss.java b/src/main/java/org/apache/sysml/lops/WeightedSquaredLoss.java
index 59b7630..77c99ec 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedSquaredLoss.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedSquaredLoss.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedSquaredLossR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedSquaredLossR.java b/src/main/java/org/apache/sysml/lops/WeightedSquaredLossR.java
index dcf87a7..53aaaae 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedSquaredLossR.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedSquaredLossR.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedUnaryMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedUnaryMM.java b/src/main/java/org/apache/sysml/lops/WeightedUnaryMM.java
index 48269df..b271fb6 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedUnaryMM.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedUnaryMM.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.Unary.OperationTypes;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.Unary.OperationTypes;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/WeightedUnaryMMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/WeightedUnaryMMR.java b/src/main/java/org/apache/sysml/lops/WeightedUnaryMMR.java
index 362cb4d..8b6f5b7 100644
--- a/src/main/java/org/apache/sysml/lops/WeightedUnaryMMR.java
+++ b/src/main/java/org/apache/sysml/lops/WeightedUnaryMMR.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.Unary.OperationTypes;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.Unary.OperationTypes;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/ZeroOut.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/ZeroOut.java b/src/main/java/org/apache/sysml/lops/ZeroOut.java
index 6494c3e..f833d8a 100644
--- a/src/main/java/org/apache/sysml/lops/ZeroOut.java
+++ b/src/main/java/org/apache/sysml/lops/ZeroOut.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class ZeroOut extends Lop
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index e5b63f5..c3881f7 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.lops.compile;
+package org.apache.sysml.lops.compile;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,56 +30,56 @@ import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.AppendM;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Data.OperationTypes;
-import com.ibm.bi.dml.lops.FunctionCallCP;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.Lop.Type;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.OutputParameters;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.SPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.AppendM;
+import org.apache.sysml.lops.BinaryM;
+import org.apache.sysml.lops.CombineBinary;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.PMMJ;
+import org.apache.sysml.lops.ParameterizedBuiltin;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.lops.Data.OperationTypes;
+import org.apache.sysml.lops.FunctionCallCP;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.Lop.Type;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.OutputParameters;
+import org.apache.sysml.lops.OutputParameters.Format;
+import org.apache.sysml.lops.PickByCount;
+import org.apache.sysml.lops.Unary;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.InstructionParser;
+import org.apache.sysml.runtime.instructions.SPInstructionParser;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.sort.PickFromCompactInputFormat;
@@ -2775,12 +2775,12 @@ public class Dag<N extends Lop>
} else if (node.getType() == Type.CombineBinary) {
// Output format of CombineBinary (CB) depends on how the output is consumed
CombineBinary combine = (CombineBinary) node;
- if ( combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreSort ) {
+ if ( combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreSort ) {
oinfo = OutputInfo.OutputInfoForSortInput;
}
- else if ( combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreCentralMoment
- || combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreCovUnweighted
- || combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreGroupedAggUnweighted ) {
+ else if ( combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCentralMoment
+ || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCovUnweighted
+ || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreGroupedAggUnweighted ) {
oinfo = OutputInfo.WeightedPairOutputInfo;
}
} else if ( node.getType() == Type.CombineTernary) {
@@ -2861,7 +2861,7 @@ public class Dag<N extends Lop>
out.addLastInstruction(currInstr);
}
else if(node instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)node).getOp() == com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM) {
+ && ((ParameterizedBuiltin)node).getOp() == org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM) {
ParameterizedBuiltin pbi = (ParameterizedBuiltin)node;
Lop input = pbi.getNamedInput(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_DATA);
@@ -3661,7 +3661,7 @@ public class Dag<N extends Lop>
if (node.getExecLocation() == ExecLocation.Data ) {
if ( ((Data)node).getFileFormatType() == FileFormatTypes.CSV
&& !(node.getInputs().get(0) instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)node.getInputs().get(0)).getOp() == com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM)) {
+ && ((ParameterizedBuiltin)node.getInputs().get(0)).getOp() == org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM)) {
// Generate write instruction, which goes into CSV_WRITE Job
int output_index = start_index[0];
shuffleInstructions.add(node.getInstructions(inputIndices.get(0), output_index));
@@ -3701,7 +3701,7 @@ public class Dag<N extends Lop>
break;
case ParameterizedBuiltin:
- if( ((ParameterizedBuiltin)node).getOp() == com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
+ if( ((ParameterizedBuiltin)node).getOp() == org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
shuffleInstructions.add(node.getInstructions(output_index));
if(DMLScript.ENABLE_DEBUG_MODE) {
MRJobLineNumbers.add(node._beginLine);
@@ -4239,13 +4239,13 @@ public class Dag<N extends Lop>
// InputInfo of L is the ouputInfo of CombineBinary
// And, the outputInfo of CombineBinary depends on the operation!
CombineBinary combine = (CombineBinary) node;
- if ( combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreSort ) {
+ if ( combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreSort ) {
nodeInputInfo = new InputInfo(SequenceFileInputFormat.class,
DoubleWritable.class, IntWritable.class);
}
- else if ( combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreCentralMoment
- || combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreCovUnweighted
- || combine.getOperation() == com.ibm.bi.dml.lops.CombineBinary.OperationTypes.PreGroupedAggUnweighted ) {
+ else if ( combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCentralMoment
+ || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCovUnweighted
+ || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreGroupedAggUnweighted ) {
nodeInputInfo = InputInfo.WeightedPairInputInfo;
}
} else if ( node.getType() == Type.CombineTernary ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/compile/JobType.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/JobType.java b/src/main/java/org/apache/sysml/lops/compile/JobType.java
index f8ce6f1..e6c6e9c 100644
--- a/src/main/java/org/apache/sysml/lops/compile/JobType.java
+++ b/src/main/java/org/apache/sysml/lops/compile/JobType.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops.compile;
+package org.apache.sysml.lops.compile;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.ParameterizedBuiltin;
+import org.apache.sysml.runtime.DMLRuntimeException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/compile/LopComparator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/LopComparator.java b/src/main/java/org/apache/sysml/lops/compile/LopComparator.java
index c5099af..01395f7 100644
--- a/src/main/java/org/apache/sysml/lops/compile/LopComparator.java
+++ b/src/main/java/org/apache/sysml/lops/compile/LopComparator.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.lops.compile;
+package org.apache.sysml.lops.compile;
import java.util.Comparator;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.lops.Lop;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index 804fe2f..5d68ec5 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -15,62 +15,62 @@
*
*/
-package com.ibm.bi.dml.lops.runtime;
+package org.apache.sysml.lops.runtime;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.io.MatrixWriterFactory;
-import com.ibm.bi.dml.runtime.matrix.CMCOVMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CombineMR;
-import com.ibm.bi.dml.runtime.matrix.DataGenMR;
-import com.ibm.bi.dml.runtime.matrix.DataPartitionMR;
-import com.ibm.bi.dml.runtime.matrix.GMR;
-import com.ibm.bi.dml.runtime.matrix.GroupedAggMR;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MMCJMR;
-import com.ibm.bi.dml.runtime.matrix.MMRJMR;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.ReblockMR;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.WriteCSVMR;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
-import com.ibm.bi.dml.runtime.transform.DataTransform;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
+import org.apache.sysml.runtime.io.MatrixWriter;
+import org.apache.sysml.runtime.io.MatrixWriterFactory;
+import org.apache.sysml.runtime.matrix.CMCOVMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CombineMR;
+import org.apache.sysml.runtime.matrix.DataGenMR;
+import org.apache.sysml.runtime.matrix.DataPartitionMR;
+import org.apache.sysml.runtime.matrix.GMR;
+import org.apache.sysml.runtime.matrix.GroupedAggMR;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MMCJMR;
+import org.apache.sysml.runtime.matrix.MMRJMR;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.ReblockMR;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.matrix.WriteCSVMR;
+import org.apache.sysml.runtime.matrix.data.LibMatrixDatagen;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.RandomMatrixGenerator;
+import org.apache.sysml.runtime.transform.DataTransform;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
public class RunMRJobs
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index d392357..ead9451 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.antlr4.DMLParserWrapper;
-import com.ibm.bi.dml.parser.python.PyDMLParserWrapper;
+import org.apache.sysml.parser.antlr4.DMLParserWrapper;
+import org.apache.sysml.parser.python.PyDMLParserWrapper;
/**
* Base class for all dml parsers in order to make the various compilation chains
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AssignmentStatement.java b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java
index 870cbe3..b343ac0 100644
--- a/src/main/java/org/apache/sysml/parser/AssignmentStatement.java
+++ b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DMLBreakpointManager;
public class AssignmentStatement extends Statement
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BinaryExpression.java b/src/main/java/org/apache/sysml/parser/BinaryExpression.java
index 38f7fb9..6ae2f99 100644
--- a/src/main/java/org/apache/sysml/parser/BinaryExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BinaryExpression.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/BooleanExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BooleanExpression.java b/src/main/java/org/apache/sysml/parser/BooleanExpression.java
index 30f154d..dc9e8fb 100644
--- a/src/main/java/org/apache/sysml/parser/BooleanExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BooleanExpression.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java
index ac5cb92..7fb76e3 100644
--- a/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index 2f39797..31886e2 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
public class BuiltinFunctionExpression extends DataIdentifier
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
index 0b76aaa..685636f 100644
--- a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
+++ b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
public class ConditionalPredicate
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ConstIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ConstIdentifier.java b/src/main/java/org/apache/sysml/parser/ConstIdentifier.java
index 947484e..dc9c0c8 100644
--- a/src/main/java/org/apache/sysml/parser/ConstIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/ConstIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
public abstract class ConstIdentifier extends Identifier
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DMLParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLParseException.java b/src/main/java/org/apache/sysml/parser/DMLParseException.java
index 8baad0c..6d138dd 100644
--- a/src/main/java/org/apache/sysml/parser/DMLParseException.java
+++ b/src/main/java/org/apache/sysml/parser/DMLParseException.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DMLProgram.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLProgram.java b/src/main/java/org/apache/sysml/parser/DMLProgram.java
index 104c231..885972d 100644
--- a/src/main/java/org/apache/sysml/parser/DMLProgram.java
+++ b/src/main/java/org/apache/sysml/parser/DMLProgram.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,27 +24,27 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index ed6f955..bc664ba 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
@@ -25,45 +25,45 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.OpOp3;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.ipa.InterProceduralAnalysis;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.FormatType;
-import com.ibm.bi.dml.parser.Expression.ParameterizedBuiltinFunctionOp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.FunctionOp.FunctionType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.Hop.OpOp3;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.MemoTable;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.ipa.InterProceduralAnalysis;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.FormatType;
+import org.apache.sysml.parser.Expression.ParameterizedBuiltinFunctionOp;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.PrintStatement.PRINTTYPE;
+import org.apache.sysml.runtime.DMLRuntimeException;
public class DMLTranslator
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index d64910e..6bbc3c8 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.BufferedReader;
import java.io.IOException;
@@ -30,12 +30,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONObject;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.JSONHelper;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.JSONHelper;
public class DataExpression extends DataIdentifier
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DataIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataIdentifier.java b/src/main/java/org/apache/sysml/parser/DataIdentifier.java
index 709be3f..cd9e452 100644
--- a/src/main/java/org/apache/sysml/parser/DataIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/DataIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
public class DataIdentifier extends Identifier
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/DoubleIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DoubleIdentifier.java b/src/main/java/org/apache/sysml/parser/DoubleIdentifier.java
index 257f652..9e2e724 100644
--- a/src/main/java/org/apache/sysml/parser/DoubleIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/DoubleIdentifier.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 478b099..af42375 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,7 +23,7 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
public abstract class Expression
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java
index 65bd878..ef363a8 100644
--- a/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java
+++ b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ForStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ForStatement.java b/src/main/java/org/apache/sysml/parser/ForStatement.java
index b6c6ce4..ce44138 100644
--- a/src/main/java/org/apache/sysml/parser/ForStatement.java
+++ b/src/main/java/org/apache/sysml/parser/ForStatement.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
index 0e4c05c..d3462ea 100644
--- a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
public class ForStatementBlock extends StatementBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/parser/FunctionCallIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/FunctionCallIdentifier.java b/src/main/java/org/apache/sysml/parser/FunctionCallIdentifier.java
index 2e67f01..a988270 100644
--- a/src/main/java/org/apache/sysml/parser/FunctionCallIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/FunctionCallIdentifier.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.parser;
+package org.apache.sysml.parser;
import java.util.ArrayList;
import java.util.HashMap;
[72/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index ebbd53d..3c1b8f8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageRuntimeException;
/**
* CP external function program block, that overcomes the need for
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
index c40420c..d962882 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
public class ForProgramBlock extends ProgramBlock
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
index 9785a38..42f917c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.utils.Statistics;
public class FunctionProgramBlock extends ProgramBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
index 925b426..4b931b5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
public class IfProgramBlock extends ProgramBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
index 5c5132d..76cb77e 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LineageObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.spark.data.LineageObject;
import java.util.HashMap;
import java.util.StringTokenizer;
@@ -37,7 +37,7 @@ import java.util.Set;
public class LocalVariableMap implements Cloneable
{
private static String eol = System.getProperty ("line.separator");
- private static String ELEMENT_DELIM = com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter.ELEMENT_DELIM;
+ private static String ELEMENT_DELIM = org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter.ELEMENT_DELIM;
private static IDSequence _seq = new IDSequence();
private HashMap <String, Data> localMap = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index faf85c9..cb5c9a6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -30,78 +30,78 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerLocal;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerRemoteMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.LocalParWorker;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.LocalTaskQueue;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ParForBody;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteDPParForMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteDPParForSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForJobReturn;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalAutomatic;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalFile;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalMemory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeRemoteMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeRemoteSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoring;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmax;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmin;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFixedsize;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerNaive;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerStatic;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimator;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTree;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTreeConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizationWrapper;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.ProgramRecompiler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.VariableSet;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitioner;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteMR;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark;
+import org.apache.sysml.runtime.controlprogram.parfor.LocalParWorker;
+import org.apache.sysml.runtime.controlprogram.parfor.LocalTaskQueue;
+import org.apache.sysml.runtime.controlprogram.parfor.ParForBody;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.RemoteDPParForMR;
+import org.apache.sysml.runtime.controlprogram.parfor.RemoteDPParForSpark;
+import org.apache.sysml.runtime.controlprogram.parfor.RemoteParForJobReturn;
+import org.apache.sysml.runtime.controlprogram.parfor.RemoteParForMR;
+import org.apache.sysml.runtime.controlprogram.parfor.RemoteParForSpark;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMerge;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeLocalAutomatic;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeLocalFile;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeLocalMemory;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeRemoteMR;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeRemoteSpark;
+import org.apache.sysml.runtime.controlprogram.parfor.Task;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitioner;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerFactoring;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmax;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmin;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerFixedsize;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerNaive;
+import org.apache.sysml.runtime.controlprogram.parfor.TaskPartitionerStatic;
+import org.apache.sysml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimator;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptTree;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptTreeConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizationWrapper;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.ProgramRecompiler;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java b/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java
index 08d7958..02e5d9c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
public class Program
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index b0cb2af..4d7a3cf 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -15,37 +15,37 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
public class ProgramBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
index cea071c..131cddc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram;
+package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
public class WhileProgramBlock extends ProgramBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/ByteBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/ByteBuffer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/ByteBuffer.java
index 97b6a18..6df3577 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/ByteBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/ByteBuffer.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.DataOutput;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.LocalFileUtils;
/**
* Wrapper for WriteBuffer byte array per matrix in order to
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
index f2802a3..01f7dcf 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.DataInput;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataInput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
public class CacheDataInput implements DataInput, MatrixBlockDataInput
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
index d821319..ce13cff 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.DataOutput;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataOutput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.matrix.data.MatrixBlockDataOutput;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
/**
* Customer DataOutput to serialize directly into the given byte array.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheException.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheException.java
index 3516b8b..cc05cbc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheException.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLRuntimeException;
public class CacheException extends DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheIOException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheIOException.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheIOException.java
index 6efda8d..edcc16d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheIOException.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheIOException.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
public class CacheIOException extends CacheException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
index e12b586..d7ca027 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.util.concurrent.atomic.AtomicLong;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatusException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatusException.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatusException.java
index 1cefff7..9783087 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatusException.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatusException.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
public class CacheStatusException extends CacheException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index 2785e64..b98a5d0 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.File;
import java.io.IOException;
@@ -23,15 +23,15 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.LazyWriteBuffer.RPolicy;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer.RPolicy;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.LocalFileUtils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/LazyWriteBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/LazyWriteBuffer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/LazyWriteBuffer.java
index 3d60788..ea76061 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/LazyWriteBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/LazyWriteBuffer.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.LocalFileUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index 839a830..981bee2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -15,38 +15,38 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.io.IOException;
import java.lang.ref.SoftReference;
import org.apache.commons.lang.mutable.MutableBoolean;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BroadcastObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDProperties;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/PageCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/PageCache.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/PageCache.java
index 2219f58..d60f2fa 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/PageCache.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/PageCache.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.caching;
+package org.apache.sysml.runtime.controlprogram.caching;
import java.lang.ref.SoftReference;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 8bdfed8..3cdb380 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -15,37 +15,37 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.context;
+package org.apache.sysml.runtime.controlprogram.context;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLFrame;
-import com.ibm.bi.dml.debug.DMLProgramCounter;
-import com.ibm.bi.dml.debug.DebugState;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DMLFrame;
+import org.apache.sysml.debug.DMLProgramCounter;
+import org.apache.sysml.debug.DebugState;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ExecutionContext
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
index 62b517c..e5c4a98 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.context;
+package org.apache.sysml.runtime.controlprogram.context;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.controlprogram.Program;
public class ExecutionContextFactory
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 48b4699..e4a5a5f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.context;
+package org.apache.sysml.runtime.controlprogram.context;
import java.io.IOException;
import java.util.LinkedList;
@@ -34,38 +34,38 @@ import org.apache.spark.storage.StorageLevel;
import scala.Tuple2;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.spark.CheckpointSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BlockPartitioner;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BroadcastObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LineageObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockPairFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyTextInputFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.Checkpoint;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.spark.CheckpointSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.runtime.instructions.spark.data.BlockPartitioner;
+import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
+import org.apache.sysml.runtime.instructions.spark.data.LineageObject;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockPairFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyTextInputFunction;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
public class SparkExecutionContext extends ExecutionContext
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
index a5c05e0..2ccb5af 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
index b3a5075..7ccbbf7 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.BufferedWriter;
import java.io.File;
@@ -37,24 +37,24 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.Cell;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.util.Cell;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.LocalFileUtils;
/**
* Partitions a given matrix into row or column partitions with a two pass-approach.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
index d0cb7fc..ab67403 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
@@ -24,20 +24,20 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
/**
* MR job class for submitting parfor remote partitioning MR jobs.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
index a78739c..83da286 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
@@ -26,19 +26,19 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* Remote data partitioner mapper implementation that does the actual
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
index 27d858f..e33a62c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -32,13 +32,13 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
/**
* Remote data partitioner reducer implementation that takes the given
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
index 41bda63..a917b05 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
/**
* MR job class for submitting parfor remote partitioning MR jobs.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
index 6210c94..0e049b9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.LinkedList;
@@ -24,16 +24,16 @@ import java.util.List;
import org.apache.hadoop.io.Writable;
import org.apache.spark.api.java.function.PairFlatMapFunction;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
import scala.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
index 8207d8a..bdd0041 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.File;
import java.util.Iterator;
@@ -27,13 +27,13 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.spark.api.java.function.VoidFunction;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
import scala.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
index 08c4733..9a0e4d1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.Collection;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
/**
* Instances of this class can be used to execute tasks in parallel. Within each ParWorker
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
index 859044a..9151b59 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.LinkedList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParForBody.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParForBody.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParForBody.java
index d60e72b..a5bbfbd 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParForBody.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParForBody.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
/**
* Wrapper for exchanging parfor body data structures.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
index 6be0527..4f9eb38 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
import java.util.List;
@@ -23,17 +23,17 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* Super class for master/worker pattern implementations. Central place to
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index 1077d3e..70560e5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
import java.util.Collection;
@@ -28,60 +28,60 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
/**
* Static functionalities for
[42/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/Hop.java b/src/main/java/com/ibm/bi/dml/hops/Hop.java
deleted file mode 100644
index 3545ba3..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/Hop.java
+++ /dev/null
@@ -1,1882 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.CSVReBlock;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-public abstract class Hop
-{
-
- protected static final Log LOG = LogFactory.getLog(Hop.class.getName());
-
- public static final long CPThreshold = 2000;
- protected static final boolean BREAKONSCALARS = false;
- protected static final boolean SPLITLARGEMATRIXMULT = true;
-
- public enum VisitStatus {
- DONE,
- VISITING,
- NOTVISITED,
- }
-
- /**
- * Optional hop interface, to be implemented by multi-threaded hops.
- */
- public interface MultiThreadedHop {
- public abstract void setMaxNumThreads( int k );
- public abstract int getMaxNumThreads();
- }
-
- // static variable to assign an unique ID to every hop that is created
- private static IDSequence _seqHopID = new IDSequence();
-
- protected long _ID;
- protected String _name;
- protected DataType _dataType;
- protected ValueType _valueType;
- protected VisitStatus _visited = VisitStatus.NOTVISITED;
- protected long _dim1 = -1;
- protected long _dim2 = -1;
- protected long _rows_in_block = -1;
- protected long _cols_in_block = -1;
- protected long _nnz = -1;
-
- protected ArrayList<Hop> _parent = new ArrayList<Hop>();
- protected ArrayList<Hop> _input = new ArrayList<Hop>();
-
- protected ExecType _etype = null; //currently used exec type
- protected ExecType _etypeForced = null; //exec type forced via platform or external optimizer
-
- // Estimated size for the output produced from this Hop
- protected double _outputMemEstimate = OptimizerUtils.INVALID_SIZE;
-
- // Estimated size for the entire operation represented by this Hop
- // It includes the memory required for all inputs as well as the output
- protected double _memEstimate = OptimizerUtils.INVALID_SIZE;
- protected double _processingMemEstimate = 0;
- protected double _spBroadcastMemEstimate = 0;
-
- // indicates if there are unknowns during compilation
- // (in that case re-complication ensures robustness and efficiency)
- protected boolean _requiresRecompile = false;
-
- // indicates if the output of this hop needs to be reblocked
- // (usually this happens on persistent reads dataops)
- protected boolean _requiresReblock = false;
-
- // indicates if the output of this hop needs to be checkpointed (cached)
- // (the default storage level for caching is not yet exposed here)
- protected boolean _requiresCheckpoint = false;
-
- // indicates if the output of this hops needs to contain materialized empty blocks
- // if those exists; otherwise only blocks w/ non-zero values are materialized
- protected boolean _outputEmptyBlocks = true;
-
- private Lop _lops = null;
-
- protected Hop(){
- //default constructor for clone
- }
-
- public Hop(String l, DataType dt, ValueType vt) {
- _ID = getNextHopID();
- setName(l);
- setDataType(dt);
- setValueType(vt);
- }
-
-
- private static long getNextHopID() {
- return _seqHopID.getNextID();
- }
-
- public long getHopID() {
- return _ID;
- }
-
- public ExecType getExecType()
- {
- return _etype;
- }
-
- public void resetExecType()
- {
- _etype = null;
- }
-
- /**
- *
- * @return
- */
- public ExecType getForcedExecType()
- {
- return _etypeForced;
- }
-
- /**
- *
- * @param etype
- */
- public void setForcedExecType(ExecType etype)
- {
- _etypeForced = etype;
- }
-
- /**
- *
- * @return
- */
- public abstract boolean allowsAllExecTypes();
-
- /**
- * Defines if this operation is transpose-safe, which means that
- * the result of op(input) is equivalent to op(t(input)).
- * Usually, this applies to aggregate operations with fixed output
- * dimension. Finally, this information is very useful in order to
- * safely optimize the plan for sparse vectors, which otherwise
- * would be (currently) always represented dense.
- *
- *
- * @return
- */
- public boolean isTransposeSafe()
- {
- //by default: its conservatively define as unsafe
- return false;
- }
-
- /**
- *
- */
- public void checkAndSetForcedPlatform()
- {
- if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE )
- _etypeForced = ExecType.CP;
- else if ( DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP )
- _etypeForced = ExecType.MR;
- else if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK )
- _etypeForced = ExecType.SPARK;
- }
-
- /**
- *
- */
- public void checkAndSetInvalidCPDimsAndSize()
- {
- if( _etype == ExecType.CP )
- {
- boolean invalid = false;
-
- //Step 1: check dimensions of output and all inputs (INTEGER)
- invalid |= !OptimizerUtils.isValidCPDimensions(_dim1, _dim2);
- for( Hop in : getInput() )
- invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2);
-
- //Step 2: check valid output and input sizes for cp (<16GB for DENSE)
- //(if the memory estimate is smaller than max_numcells we are guaranteed to have it in sparse representation)
- invalid |= !( OptimizerUtils.isValidCPMatrixSize(_dim1, _dim2, OptimizerUtils.getSparsity(_dim1, _dim2, _nnz))
- || getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE );
- for( Hop in : getInput() )
- invalid |= !( OptimizerUtils.isValidCPMatrixSize(in._dim1, in._dim2, OptimizerUtils.getSparsity(in._dim1, in._dim2, in._nnz))
- || in.getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE);
-
- //force exec type mr if necessary
- if( invalid ) {
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID )
- _etype = ExecType.MR;
- else if( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
- _etype = ExecType.SPARK;
- }
- }
- }
-
- public void setRequiresReblock(boolean flag)
- {
- _requiresReblock = flag;
- }
-
- public boolean hasMatrixInputWithDifferentBlocksizes()
- {
- for( Hop c : getInput() ) {
- if( c.getDataType()==DataType.MATRIX
- &&(getRowsInBlock() != c.getRowsInBlock()
- || getColsInBlock() != c.getColsInBlock()) )
- {
- return true;
- }
- }
-
- return false;
- }
-
- public void setOutputBlocksizes( long brlen, long bclen )
- {
- setRowsInBlock( brlen );
- setColsInBlock( bclen );
- }
-
- public boolean requiresReblock()
- {
- return _requiresReblock;
- }
-
- public void setRequiresCheckpoint(boolean flag)
- {
- _requiresCheckpoint = flag;
- }
-
- public boolean requiresCheckpoint()
- {
- return _requiresCheckpoint;
- }
-
-
- /**
- *
- * @throws HopsException
- */
- public void constructAndSetLopsDataFlowProperties()
- throws HopsException
- {
- //Step 1: construct reblock lop if required (output of hop)
- constructAndSetReblockLopIfRequired();
-
- //Step 2: construct checkpoint lop if required (output of hop or reblock)
- constructAndSetCheckpointLopIfRequired();
- }
-
- /**
- *
- * @throws HopsException
- */
- private void constructAndSetReblockLopIfRequired()
- throws HopsException
- {
- //determine execution type
- ExecType et = ExecType.CP;
- if( DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE
- && !(getDataType()==DataType.SCALAR) )
- {
- et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
- }
-
- //add reblock lop to output if required
- if( _requiresReblock && et != ExecType.CP )
- {
- Lop input = getLops();
- Lop reblock = null;
-
- try
- {
- if( (this instanceof DataOp // CSV
- && ((DataOp)this).getDataOpType() == DataOpTypes.PERSISTENTREAD
- && ((DataOp)this).getInputFormatType() == FileFormatTypes.CSV )
- || (this instanceof ParameterizedBuiltinOp
- && ((ParameterizedBuiltinOp)this).getOp() == ParamBuiltinOp.TRANSFORM) )
- {
- reblock = new CSVReBlock( input, getRowsInBlock(), getColsInBlock(),
- getDataType(), getValueType(), et);
- }
- else //TEXT / MM / BINARYBLOCK / BINARYCELL
- {
- reblock = new ReBlock( input, getRowsInBlock(), getColsInBlock(),
- getDataType(), getValueType(), _outputEmptyBlocks, et);
- }
- }
- catch( LopsException ex ) {
- throw new HopsException(ex);
- }
-
- setOutputDimensions( reblock );
- setLineNumbers( reblock );
- setLops( reblock );
- }
- }
-
- /**
- *
- * @throws HopsException
- */
- private void constructAndSetCheckpointLopIfRequired()
- throws HopsException
- {
- //determine execution type
- ExecType et = ExecType.CP;
- if( OptimizerUtils.isSparkExecutionMode()
- && getDataType()!=DataType.SCALAR )
- {
- //conditional checkpoint based on memory estimate in order to
- //(1) avoid unnecessary persist and unpersist calls, and
- //(2) avoid unnecessary creation of spark context (incl executors)
- if( OptimizerUtils.isHybridExecutionMode()
- && (getDim2() > 1 && _outputMemEstimate < OptimizerUtils.getLocalMemBudget()
- || getDim2() == 1 && _outputMemEstimate < OptimizerUtils.getLocalMemBudget()/3 )
- || _etypeForced == ExecType.CP )
- {
- et = ExecType.CP;
- }
- else //default case
- {
- et = ExecType.SPARK;
- }
- }
-
- //add checkpoint lop to output if required
- if( _requiresCheckpoint && et != ExecType.CP )
- {
- try
- {
- //investigate need for serialized storage of large sparse matrices
- //(compile- instead of runtime-level for better debugging)
- boolean serializedStorage = false;
- if( dimsKnown(true) ) {
- double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
- double dataCache = SparkExecutionContext.getConfiguredTotalDataMemory(true);
- serializedStorage = (MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz)
- && matrixPSize > dataCache ); //sparse in-memory does not fit in agg mem
- }
- else {
- setRequiresRecompile();
- }
-
- //construct checkpoint w/ right storage level
- Lop input = getLops();
- Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(),
- serializedStorage ? Checkpoint.getSerializeStorageLevelString() :
- Checkpoint.getDefaultStorageLevelString() );
-
- setOutputDimensions( chkpoint );
- setLineNumbers( chkpoint );
- setLops( chkpoint );
- }
- catch( LopsException ex ) {
- throw new HopsException(ex);
- }
- }
-
- }
-
-
- /**
- *
- * @param inputPos
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- public static Lop createOffsetLop( Hop hop, boolean repCols )
- throws HopsException, LopsException
- {
- Lop offset = null;
-
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && hop.dimsKnown() )
- {
- // If dynamic recompilation is enabled and dims are known, we can replace the ncol with
- // a literal in order to increase the piggybacking potential. This is safe because append
- // is always marked for recompilation and hence, we have propagated the exact dimensions.
- offset = Data.createLiteralLop(ValueType.INT, String.valueOf(repCols ? hop.getDim2() : hop.getDim1()));
- }
- else
- {
- offset = new UnaryCP(hop.constructLops(),
- repCols ? UnaryCP.OperationTypes.NCOL : UnaryCP.OperationTypes.NROW,
- DataType.SCALAR, ValueType.INT);
- }
-
- offset.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- offset.setAllPositions(hop.getBeginLine(), hop.getBeginColumn(), hop.getEndLine(), hop.getEndColumn());
-
- return offset;
- }
-
- public void setOutputEmptyBlocks(boolean flag)
- {
- _outputEmptyBlocks = flag;
- }
-
- public boolean isOutputEmptyBlocks()
- {
- return _outputEmptyBlocks;
- }
-
- /**
- * Returns the memory estimate for the output produced from this Hop.
- * It must be invoked only within Hops. From outside Hops, one must
- * only use getMemEstimate(), which gives memory required to store
- * all inputs and the output.
- *
- * @return
- */
- protected double getOutputSize()
- {
- return _outputMemEstimate;
- }
-
- /**
- *
- * @return
- */
- protected double getInputSize()
- {
- double sum = 0;
- int len = _input.size();
-
- for( int i=0; i<len; i++ ) //for all inputs
- {
- Hop hi = _input.get(i);
- double hmout = hi.getOutputMemEstimate();
-
- if( hmout > 1024*1024 ) {//for relevant sizes
- //check if already included in estimate (if an input is used
- //multiple times it is still only required once in memory)
- //(not that this check benefits from common subexpression elimination)
- boolean flag = false;
- for( int j=0; j<i; j++ )
- flag |= (hi == _input.get(j));
- hmout = flag ? 0 : hmout;
- }
-
- sum += hmout;
- }
-
- //for(Hop h : _input ) {
- // sum += h._outputMemEstimate;
- //}
-
- return sum;
- }
-
- /**
- *
- * @return
- */
- protected double getInputOutputSize()
- {
- double sum = 0;
- sum += _outputMemEstimate;
- sum += _processingMemEstimate;
- sum += getInputSize();
-
- return sum;
- }
-
- /**
- *
- * @param pos
- * @return
- */
- protected double getInputSize( int pos ){
- double ret = 0;
- if( _input.size()>pos )
- ret = _input.get(pos)._outputMemEstimate;
- return ret;
- }
-
- protected double getIntermediateSize() {
- return _processingMemEstimate;
- }
-
- /**
- * NOTES:
- * * Purpose: Whenever the output dimensions / sparsity of a hop are unknown, this hop
- * should store its worst-case output statistics (if known) in that table. Subsequent
- * hops can then
- * * Invocation: Intended to be called for ALL root nodes of one Hops DAG with the same
- * (initially empty) memo table.
- *
- * @param memo
- * @return
- */
- public double getMemEstimate()
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- if ( ! isMemEstimated() ) {
- //LOG.warn("Nonexisting memory estimate - reestimating w/o memo table.");
- computeMemEstimate( new MemoTable() );
- }
- return _memEstimate;
- }
- else {
- return OptimizerUtils.INVALID_SIZE;
- }
- }
-
- /**
- * Returns memory estimate in bytes
- *
- * @param mem
- */
- public void setMemEstimate( double mem )
- {
- _memEstimate = mem;
- }
-
- public void clearMemEstimate()
- {
- _memEstimate = OptimizerUtils.INVALID_SIZE;
- }
-
- public boolean isMemEstimated()
- {
- return (_memEstimate != OptimizerUtils.INVALID_SIZE);
- }
-
- //wrappers for meaningful public names to memory estimates.
-
- public double getInputMemEstimate()
- {
- return getInputSize();
- }
-
- public double getOutputMemEstimate()
- {
- return getOutputSize();
- }
-
- public double getIntermediateMemEstimate()
- {
- return getIntermediateSize();
- }
-
- public double getSpBroadcastSize()
- {
- return _spBroadcastMemEstimate;
- }
-
- /**
- * Computes the estimate of memory required to store the input/output of this hop in memory.
- * This is the default implementation (orchestration of hop-specific implementation)
- * that should suffice for most hops. If a hop requires more control, this method should
- * be overwritten with awareness of (1) output estimates, and (2) propagation of worst-case
- * matrix characteristics (dimensions, sparsity).
- *
- * TODO remove memo table and, on constructor refresh, inference in refresh, single compute mem,
- * maybe general computeMemEstimate, flags to indicate if estimate or not.
- *
- * @return computed estimate
- */
- public void computeMemEstimate( MemoTable memo )
- {
- long[] wstats = null;
-
- ////////
- //Step 1) Compute hop output memory estimate (incl size inference)
-
- switch( getDataType() )
- {
- case SCALAR:
- {
- //memory estimate always known
- if( getValueType()== ValueType.DOUBLE) //default case
- _outputMemEstimate = OptimizerUtils.DOUBLE_SIZE;
- else //literalops, dataops
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, _nnz );
- break;
- }
- case MATRIX:
- {
- //1a) mem estimate based on exactly known dimensions and sparsity
- if( dimsKnown(true) ) {
- //nnz always exactly known (see dimsKnown(true))
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, _nnz );
- }
- //1b) infer output statistics and mem estimate based on these statistics
- else if( memo.hasInputStatistics(this) )
- {
- //infer the output stats
- wstats = inferOutputCharacteristics(memo);
-
- if( wstats != null ) {
- //use worst case characteristics to estimate mem
- long lnnz = ((wstats[2]>=0)?wstats[2]:wstats[0]*wstats[1]);
- _outputMemEstimate = computeOutputMemEstimate( wstats[0], wstats[1], lnnz );
-
- //propagate worst-case estimate
- memo.memoizeStatistics(getHopID(), wstats[0], wstats[1], wstats[2]);
- }
- else if( dimsKnown() ) {
- //nnz unknown, estimate mem as dense
- long lnnz = _dim1*_dim2;
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- }
- else {
- //unknown output size
- _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
- }
- }
- //1c) mem estimate based on exactly known dimensions and unknown sparsity
- //(required e.g., for datagenops w/o any input statistics)
- else if( dimsKnown() ) {
- //nnz unknown, estimate mem as dense
- long lnnz = _dim1*_dim2;
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- }
- //1d) fallback: unknown output size
- else {
- _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
- }
-
- break;
- }
- case OBJECT:
- case UNKNOWN:
- case FRAME:
- {
- //memory estimate always unknown
- _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
- break;
- }
- }
-
- ////////
- //Step 2) Compute hop intermediate memory estimate
-
- //note: ensure consistency w/ step 1 (for simplified debugging)
-
- if( dimsKnown(true) ) { //incl scalar output
- //nnz always exactly known (see dimsKnown(true))
- _processingMemEstimate = computeIntermediateMemEstimate( _dim1, _dim2, _nnz );
- }
- else if( wstats!=null ) {
- //use worst case characteristics to estimate mem
- long lnnz = ((wstats[2]>=0)?wstats[2]:wstats[0]*wstats[1]);
- _processingMemEstimate = computeIntermediateMemEstimate( wstats[0], wstats[1], lnnz );
- }
- else if( dimsKnown() ){
- //nnz unknown, estimate mem as dense
- long lnnz = _dim1 * _dim2;
- _processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
- }
-
-
- ////////
- //Step 3) Compute final hop memory estimate
-
- //final estimate (sum of inputs/intermediates/output)
- _memEstimate = getInputOutputSize();
- }
-
-
- /**
- * Computes the hop-specific output memory estimate in bytes. Should be 0 if not
- * applicable.
- *
- * @param dim1
- * @param dim2
- * @param nnz
- * @return
- */
- protected abstract double computeOutputMemEstimate( long dim1, long dim2, long nnz );
-
- /**
- * Computes the hop-specific intermediate memory estimate in bytes. Should be 0 if not
- * applicable.
- *
- * @param dim1
- * @param dim2
- * @param nnz
- * @return
- */
- protected abstract double computeIntermediateMemEstimate( long dim1, long dim2, long nnz );
-
- /**
- * Computes the output matrix characteristics (rows, cols, nnz) based on worst-case output
- * and/or input estimates. Should return null if dimensions are unknown.
- *
- * @param memo
- * @return
- */
- protected abstract long[] inferOutputCharacteristics( MemoTable memo );
-
-
-
- /**
- * This function is used only for sanity check.
- * Returns true if estimates for all the hops in the DAG rooted at the current
- * hop are computed. Returns false if any of the hops have INVALID estimate.
- *
- * @return
- */
- public boolean checkEstimates() {
- boolean childStatus = true;
- for (Hop h : this.getInput())
- childStatus = childStatus && h.checkEstimates();
- return childStatus && (_memEstimate != OptimizerUtils.INVALID_SIZE);
- }
-
- /**
- * Recursively computes memory estimates for all the Hops in the DAG rooted at the
- * current hop pointed by <code>this</code>.
- *
- */
- public void refreshMemEstimates( MemoTable memo ) {
- if (getVisited() == VisitStatus.DONE)
- return;
- for (Hop h : this.getInput())
- h.refreshMemEstimates( memo );
- this.computeMemEstimate( memo );
- this.setVisited(VisitStatus.DONE);
- }
-
- /**
- * This method determines the execution type (CP, MR) based ONLY on the
- * estimated memory footprint required for this operation, which includes
- * memory for all inputs and the output represented by this Hop.
- *
- * It is used when <code>OptimizationType = MEMORY_BASED</code>.
- * This optimization schedules an operation to CP whenever inputs+output
- * fit in memory -- note that this decision MAY NOT be optimal in terms of
- * execution time.
- *
- * @return
- */
- protected ExecType findExecTypeByMemEstimate() {
- ExecType et = null;
- char c = ' ';
- if ( getMemEstimate() < OptimizerUtils.getLocalMemBudget() ) {
- et = ExecType.CP;
- }
- else {
- if( DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID )
- et = ExecType.MR;
- else if( DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK )
- et = ExecType.SPARK;
-
- c = '*';
- }
-
- if (LOG.isDebugEnabled()){
- String s = String.format(" %c %-5s %-8s (%s,%s) %s", c, getHopID(), getOpString(), OptimizerUtils.toMB(_outputMemEstimate), OptimizerUtils.toMB(_memEstimate), et);
- //System.out.println(s);
- LOG.debug(s);
- }
-
- return et;
- }
-
- public ArrayList<Hop> getParent() {
- return _parent;
- }
-
- public ArrayList<Hop> getInput() {
- return _input;
- }
-
- /**
- * Create bidirectional links
- *
- * @param h
- */
- public void addInput( Hop h )
- {
- _input.add(h);
- h._parent.add(this);
- }
-
- public long getRowsInBlock() {
- return _rows_in_block;
- }
-
- public void setRowsInBlock(long rowsInBlock) {
- _rows_in_block = rowsInBlock;
- }
-
- public long getColsInBlock() {
- return _cols_in_block;
- }
-
- public void setColsInBlock(long colsInBlock) {
- _cols_in_block = colsInBlock;
- }
-
- public void setNnz(long nnz){
- _nnz = nnz;
- }
-
- public long getNnz(){
- return _nnz;
- }
-
- public abstract Lop constructLops()
- throws HopsException, LopsException;
-
- protected abstract ExecType optFindExecType()
- throws HopsException;
-
- public abstract String getOpString();
-
- protected boolean isVector() {
- return (dimsKnown() && (_dim1 == 1 || _dim2 == 1) );
- }
-
- protected boolean areDimsBelowThreshold() {
- return (dimsKnown() && _dim1 <= Hop.CPThreshold && _dim2 <= Hop.CPThreshold );
- }
-
- public boolean dimsKnown() {
- return ( _dataType == DataType.SCALAR || (_dataType==DataType.MATRIX && _dim1 > 0 && _dim2 > 0) );
- }
-
- public boolean dimsKnown(boolean includeNnz) {
- return ( _dataType == DataType.SCALAR || (_dataType==DataType.MATRIX && _dim1 > 0 && _dim2 > 0 && ((includeNnz)? _nnz>=0 : true)) );
- }
-
- public boolean dimsKnownAny() {
- return ( _dataType == DataType.SCALAR || (_dataType==DataType.MATRIX && (_dim1 > 0 || _dim2 > 0)) );
- }
-
- public static void resetVisitStatus( ArrayList<Hop> hops )
- {
- if( hops != null )
- for( Hop hopRoot : hops )
- hopRoot.resetVisitStatus();
- }
-
- public void resetVisitStatus()
- {
- if ( getVisited() == Hop.VisitStatus.NOTVISITED )
- return;
-
- for (Hop h : this.getInput())
- h.resetVisitStatus();
-
- setVisited(Hop.VisitStatus.NOTVISITED);
- }
-
- public static void resetRecompilationFlag( ArrayList<Hop> hops, ExecType et )
- {
- resetVisitStatus( hops );
- for( Hop hopRoot : hops )
- hopRoot.resetRecompilationFlag( et );
- }
-
- public static void resetRecompilationFlag( Hop hops, ExecType et )
- {
- hops.resetVisitStatus();
- hops.resetRecompilationFlag( et );
- }
-
- private void resetRecompilationFlag( ExecType et )
- {
- if( getVisited() == VisitStatus.DONE )
- return;
-
- //process child hops
- for (Hop h : getInput())
- h.resetRecompilationFlag( et );
-
- //reset recompile flag
- if( et == null || getExecType() == et || getExecType()==null )
- _requiresRecompile = false;
-
- this.setVisited(VisitStatus.DONE);
- }
-
-
- /**
- * Test and debugging only.
- *
- * @param h
- * @throws HopsException
- */
- public void checkParentChildPointers( )
- throws HopsException
- {
- if( getVisited() == VisitStatus.DONE )
- return;
-
- for( Hop in : getInput() )
- {
- if( !in.getParent().contains(this) )
- throw new HopsException("Parent-Child pointers incorrect.");
- in.checkParentChildPointers();
- }
-
- setVisited(VisitStatus.DONE);
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()) {
- StringBuilder s = new StringBuilder("");
- s.append(getClass().getSimpleName() + " " + getHopID() + "\n");
- s.append(" Label: " + getName() + "; DataType: " + _dataType + "; ValueType: " + _valueType + "\n");
- s.append(" Parent: ");
- for (Hop h : getParent()) {
- s.append(h.hashCode() + "; ");
- }
- ;
- s.append("\n Input: ");
- for (Hop h : getInput()) {
- s.append(h.getHopID() + "; ");
- }
-
- s.append("\n dims [" + _dim1 + "," + _dim2 + "] blk [" + _rows_in_block + "," + _cols_in_block + "] nnz " + _nnz);
- s.append(" MemEstimate = Out " + (_outputMemEstimate/1024/1024) + " MB, In&Out " + (_memEstimate/1024/1024) + " MB" );
- LOG.debug(s.toString());
- }
- }
-
- public long getDim1() {
- return _dim1;
- }
-
- public void setDim1(long dim1) {
- _dim1 = dim1;
- }
-
- public long getDim2() {
- return _dim2;
- }
-
- public void setDim2(long dim2) {
- _dim2 = dim2;
- }
-
- protected void setOutputDimensions(Lop lop)
- throws HopsException
- {
- lop.getOutputParameters().setDimensions(
- getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- }
-
- public Lop getLops() {
- return _lops;
- }
-
- public void setLops(Lop lops) {
- _lops = lops;
- }
-
- public VisitStatus getVisited() {
- return _visited;
- }
-
- public DataType getDataType() {
- return _dataType;
- }
-
- public void setDataType( DataType dt ) {
- _dataType = dt;
- }
-
- public void setVisited(VisitStatus visited) {
- _visited = visited;
- }
-
- public void setName(String _name) {
- this._name = _name;
- }
-
- public String getName() {
- return _name;
- }
-
- public ValueType getValueType() {
- return _valueType;
- }
-
- public void setValueType(ValueType vt) {
- _valueType = vt;
- }
-
- public enum OpOp1 {
- NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP,
- CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN,
- PRINT, EIGEN, NROW, NCOL, LENGTH, ROUND, IQM, STOP, CEIL, FLOOR, MEDIAN, INVERSE,
- //cumulative sums, products, extreme values
- CUMSUM, CUMPROD, CUMMIN, CUMMAX,
- //fused ML-specific operators for performance
- SPROP, //sample proportion: P * (1 - P)
- SIGMOID, //sigmoid function: 1 / (1 + exp(-X))
- SELP, //select positive: X * (X>0)
- }
-
- // Operations that require two operands
- public enum OpOp2 {
- PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, LESS, LESSEQUAL, GREATER, GREATEREQUAL, EQUAL, NOTEQUAL,
- MIN, MAX, AND, OR, LOG, POW, PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM,
- CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SEQINCR, SOLVE, MEDIAN, INVALID,
- //fused ML-specific operators for performance
- MINUS_NZ, //sparse-safe minus: X-(mean*ppred(X,0,!=))
- LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X,0.5)
- MINUS1_MULT, //1-X*Y
- };
-
- // Operations that require 3 operands
- public enum OpOp3 {
- QUANTILE, INTERQUANTILE, CTABLE, CENTRALMOMENT, COVARIANCE, INVALID
- };
-
- // Operations that require 4 operands
- public enum OpOp4 {
- WSLOSS, //weighted sloss mm
- WSIGMOID, //weighted sigmoid mm
- WDIVMM, //weighted divide mm
- WCEMM, //weighted cross entropy mm
- WUMM, //weighted unary mm
- INVALID
- };
-
-
- public enum AggOp {
- SUM, SUM_SQ, MIN, MAX, TRACE, PROD, MEAN, MAXINDEX, MININDEX
- };
-
- public enum ReOrgOp {
- TRANSPOSE, DIAG, RESHAPE, SORT
- //Note: Diag types are invalid because for unknown sizes this would
- //create incorrect plans (now we try to infer it for memory estimates
- //and rewrites but the final choice is made during runtime)
- //DIAG_V2M, DIAG_M2V,
- };
-
- public enum DataGenMethod {
- RAND, SEQ, SINIT, SAMPLE, INVALID
- };
-
- public enum ParamBuiltinOp {
- INVALID, CDF, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND, TRANSFORM
- };
-
- /**
- * Functions that are built in, but whose execution takes place in an
- * external library
- */
- public enum ExtBuiltInOp {
- EIGEN, CHOLESKY
- };
-
- public enum FileFormatTypes {
- TEXT, BINARY, MM, CSV
- };
-
- public enum DataOpTypes {
- PERSISTENTREAD, PERSISTENTWRITE, TRANSIENTREAD, TRANSIENTWRITE, FUNCTIONOUTPUT
- };
-
- public enum Direction {
- RowCol, Row, Col
- };
-
- protected static final HashMap<DataOpTypes, com.ibm.bi.dml.lops.Data.OperationTypes> HopsData2Lops;
- static {
- HopsData2Lops = new HashMap<Hop.DataOpTypes, com.ibm.bi.dml.lops.Data.OperationTypes>();
- HopsData2Lops.put(DataOpTypes.PERSISTENTREAD, com.ibm.bi.dml.lops.Data.OperationTypes.READ);
- HopsData2Lops.put(DataOpTypes.PERSISTENTWRITE, com.ibm.bi.dml.lops.Data.OperationTypes.WRITE);
- HopsData2Lops.put(DataOpTypes.TRANSIENTWRITE, com.ibm.bi.dml.lops.Data.OperationTypes.WRITE);
- HopsData2Lops.put(DataOpTypes.TRANSIENTREAD, com.ibm.bi.dml.lops.Data.OperationTypes.READ);
- }
-
- protected static final HashMap<Hop.AggOp, com.ibm.bi.dml.lops.Aggregate.OperationTypes> HopsAgg2Lops;
- static {
- HopsAgg2Lops = new HashMap<Hop.AggOp, com.ibm.bi.dml.lops.Aggregate.OperationTypes>();
- HopsAgg2Lops.put(AggOp.SUM, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanSum);
- HopsAgg2Lops.put(AggOp.SUM_SQ, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanSumSq);
- HopsAgg2Lops.put(AggOp.TRACE, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanTrace);
- HopsAgg2Lops.put(AggOp.MIN, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Min);
- HopsAgg2Lops.put(AggOp.MAX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Max);
- HopsAgg2Lops.put(AggOp.MAXINDEX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.MaxIndex);
- HopsAgg2Lops.put(AggOp.MININDEX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.MinIndex);
- HopsAgg2Lops.put(AggOp.PROD, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Product);
- HopsAgg2Lops.put(AggOp.MEAN, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Mean);
- }
-
- protected static final HashMap<ReOrgOp, com.ibm.bi.dml.lops.Transform.OperationTypes> HopsTransf2Lops;
- static {
- HopsTransf2Lops = new HashMap<ReOrgOp, com.ibm.bi.dml.lops.Transform.OperationTypes>();
- HopsTransf2Lops.put(ReOrgOp.TRANSPOSE, com.ibm.bi.dml.lops.Transform.OperationTypes.Transpose);
- HopsTransf2Lops.put(ReOrgOp.DIAG, com.ibm.bi.dml.lops.Transform.OperationTypes.Diag);
- HopsTransf2Lops.put(ReOrgOp.RESHAPE, com.ibm.bi.dml.lops.Transform.OperationTypes.Reshape);
- HopsTransf2Lops.put(ReOrgOp.SORT, com.ibm.bi.dml.lops.Transform.OperationTypes.Sort);
-
- }
-
- protected static final HashMap<Hop.Direction, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes> HopsDirection2Lops;
- static {
- HopsDirection2Lops = new HashMap<Hop.Direction, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes>();
- HopsDirection2Lops.put(Direction.RowCol, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.RowCol);
- HopsDirection2Lops.put(Direction.Col, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.Col);
- HopsDirection2Lops.put(Direction.Row, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.Row);
-
- }
-
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Binary.OperationTypes> HopsOpOp2LopsB;
- static {
- HopsOpOp2LopsB = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Binary.OperationTypes>();
- HopsOpOp2LopsB.put(OpOp2.PLUS, com.ibm.bi.dml.lops.Binary.OperationTypes.ADD);
- HopsOpOp2LopsB.put(OpOp2.MINUS, com.ibm.bi.dml.lops.Binary.OperationTypes.SUBTRACT);
- HopsOpOp2LopsB.put(OpOp2.MULT, com.ibm.bi.dml.lops.Binary.OperationTypes.MULTIPLY);
- HopsOpOp2LopsB.put(OpOp2.DIV, com.ibm.bi.dml.lops.Binary.OperationTypes.DIVIDE);
- HopsOpOp2LopsB.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.Binary.OperationTypes.MODULUS);
- HopsOpOp2LopsB.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.Binary.OperationTypes.INTDIV);
- HopsOpOp2LopsB.put(OpOp2.MINUS1_MULT, com.ibm.bi.dml.lops.Binary.OperationTypes.MINUS1_MULTIPLY);
- HopsOpOp2LopsB.put(OpOp2.LESS, com.ibm.bi.dml.lops.Binary.OperationTypes.LESS_THAN);
- HopsOpOp2LopsB.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.GREATER, com.ibm.bi.dml.lops.Binary.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsB.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.EQUALS);
- HopsOpOp2LopsB.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.MIN, com.ibm.bi.dml.lops.Binary.OperationTypes.MIN);
- HopsOpOp2LopsB.put(OpOp2.MAX, com.ibm.bi.dml.lops.Binary.OperationTypes.MAX);
- HopsOpOp2LopsB.put(OpOp2.AND, com.ibm.bi.dml.lops.Binary.OperationTypes.OR);
- HopsOpOp2LopsB.put(OpOp2.OR, com.ibm.bi.dml.lops.Binary.OperationTypes.AND);
- HopsOpOp2LopsB.put(OpOp2.SOLVE, com.ibm.bi.dml.lops.Binary.OperationTypes.SOLVE);
- HopsOpOp2LopsB.put(OpOp2.POW, com.ibm.bi.dml.lops.Binary.OperationTypes.POW);
- HopsOpOp2LopsB.put(OpOp2.LOG, com.ibm.bi.dml.lops.Binary.OperationTypes.NOTSUPPORTED);
- }
-
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes> HopsOpOp2LopsBS;
- static {
- HopsOpOp2LopsBS = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes>();
- HopsOpOp2LopsBS.put(OpOp2.PLUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.ADD);
- HopsOpOp2LopsBS.put(OpOp2.MINUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.SUBTRACT);
- HopsOpOp2LopsBS.put(OpOp2.MULT, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MULTIPLY);
- HopsOpOp2LopsBS.put(OpOp2.DIV, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.DIVIDE);
- HopsOpOp2LopsBS.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MODULUS);
- HopsOpOp2LopsBS.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.INTDIV);
- HopsOpOp2LopsBS.put(OpOp2.LESS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LESS_THAN);
- HopsOpOp2LopsBS.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.GREATER, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsBS.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.MIN, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MIN);
- HopsOpOp2LopsBS.put(OpOp2.MAX, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MAX);
- HopsOpOp2LopsBS.put(OpOp2.AND, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.AND);
- HopsOpOp2LopsBS.put(OpOp2.OR, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.OR);
- HopsOpOp2LopsBS.put(OpOp2.LOG, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LOG);
- HopsOpOp2LopsBS.put(OpOp2.POW, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.POW);
- HopsOpOp2LopsBS.put(OpOp2.PRINT, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.PRINT);
- HopsOpOp2LopsBS.put(OpOp2.SEQINCR, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.SEQINCR);
- }
-
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Unary.OperationTypes> HopsOpOp2LopsU;
- static {
- HopsOpOp2LopsU = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Unary.OperationTypes>();
- HopsOpOp2LopsU.put(OpOp2.PLUS, com.ibm.bi.dml.lops.Unary.OperationTypes.ADD);
- HopsOpOp2LopsU.put(OpOp2.MINUS, com.ibm.bi.dml.lops.Unary.OperationTypes.SUBTRACT);
- HopsOpOp2LopsU.put(OpOp2.MULT, com.ibm.bi.dml.lops.Unary.OperationTypes.MULTIPLY);
- HopsOpOp2LopsU.put(OpOp2.DIV, com.ibm.bi.dml.lops.Unary.OperationTypes.DIVIDE);
- HopsOpOp2LopsU.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.Unary.OperationTypes.MODULUS);
- HopsOpOp2LopsU.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.Unary.OperationTypes.INTDIV);
- HopsOpOp2LopsU.put(OpOp2.MINUS1_MULT, com.ibm.bi.dml.lops.Unary.OperationTypes.MINUS1_MULTIPLY);
- HopsOpOp2LopsU.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.LESS, com.ibm.bi.dml.lops.Unary.OperationTypes.LESS_THAN);
- HopsOpOp2LopsU.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.GREATER, com.ibm.bi.dml.lops.Unary.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsU.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.EQUALS);
- HopsOpOp2LopsU.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.AND, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp2LopsU.put(OpOp2.OR, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp2LopsU.put(OpOp2.MAX, com.ibm.bi.dml.lops.Unary.OperationTypes.MAX);
- HopsOpOp2LopsU.put(OpOp2.MIN, com.ibm.bi.dml.lops.Unary.OperationTypes.MIN);
- HopsOpOp2LopsU.put(OpOp2.LOG, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG);
- HopsOpOp2LopsU.put(OpOp2.POW, com.ibm.bi.dml.lops.Unary.OperationTypes.POW);
- HopsOpOp2LopsU.put(OpOp2.MINUS_NZ, com.ibm.bi.dml.lops.Unary.OperationTypes.SUBTRACT_NZ);
- HopsOpOp2LopsU.put(OpOp2.LOG_NZ, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG_NZ);
- }
-
- protected static final HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.Unary.OperationTypes> HopsOpOp1LopsU;
- static {
- HopsOpOp1LopsU = new HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.Unary.OperationTypes>();
- HopsOpOp1LopsU.put(OpOp1.NOT, com.ibm.bi.dml.lops.Unary.OperationTypes.NOT);
- HopsOpOp1LopsU.put(OpOp1.ABS, com.ibm.bi.dml.lops.Unary.OperationTypes.ABS);
- HopsOpOp1LopsU.put(OpOp1.SIN, com.ibm.bi.dml.lops.Unary.OperationTypes.SIN);
- HopsOpOp1LopsU.put(OpOp1.COS, com.ibm.bi.dml.lops.Unary.OperationTypes.COS);
- HopsOpOp1LopsU.put(OpOp1.TAN, com.ibm.bi.dml.lops.Unary.OperationTypes.TAN);
- HopsOpOp1LopsU.put(OpOp1.ASIN, com.ibm.bi.dml.lops.Unary.OperationTypes.ASIN);
- HopsOpOp1LopsU.put(OpOp1.ACOS, com.ibm.bi.dml.lops.Unary.OperationTypes.ACOS);
- HopsOpOp1LopsU.put(OpOp1.ATAN, com.ibm.bi.dml.lops.Unary.OperationTypes.ATAN);
- HopsOpOp1LopsU.put(OpOp1.SQRT, com.ibm.bi.dml.lops.Unary.OperationTypes.SQRT);
- HopsOpOp1LopsU.put(OpOp1.EXP, com.ibm.bi.dml.lops.Unary.OperationTypes.EXP);
- HopsOpOp1LopsU.put(OpOp1.LOG, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG);
- HopsOpOp1LopsU.put(OpOp1.ROUND, com.ibm.bi.dml.lops.Unary.OperationTypes.ROUND);
- HopsOpOp1LopsU.put(OpOp1.CEIL, com.ibm.bi.dml.lops.Unary.OperationTypes.CEIL);
- HopsOpOp1LopsU.put(OpOp1.FLOOR, com.ibm.bi.dml.lops.Unary.OperationTypes.FLOOR);
- HopsOpOp1LopsU.put(OpOp1.CUMSUM, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMSUM);
- HopsOpOp1LopsU.put(OpOp1.CUMPROD, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMPROD);
- HopsOpOp1LopsU.put(OpOp1.CUMMIN, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMMIN);
- HopsOpOp1LopsU.put(OpOp1.CUMMAX, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMMAX);
- HopsOpOp1LopsU.put(OpOp1.INVERSE, com.ibm.bi.dml.lops.Unary.OperationTypes.INVERSE);
- HopsOpOp1LopsU.put(OpOp1.CAST_AS_SCALAR, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp1LopsU.put(OpOp1.SPROP, com.ibm.bi.dml.lops.Unary.OperationTypes.SPROP);
- HopsOpOp1LopsU.put(OpOp1.SIGMOID, com.ibm.bi.dml.lops.Unary.OperationTypes.SIGMOID);
- HopsOpOp1LopsU.put(OpOp1.SELP, com.ibm.bi.dml.lops.Unary.OperationTypes.SELP);
- }
-
- protected static final HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.UnaryCP.OperationTypes> HopsOpOp1LopsUS;
- static {
- HopsOpOp1LopsUS = new HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.UnaryCP.OperationTypes>();
- HopsOpOp1LopsUS.put(OpOp1.NOT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NOT);
- HopsOpOp1LopsUS.put(OpOp1.ABS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ABS);
- HopsOpOp1LopsUS.put(OpOp1.SIN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.SIN);
- HopsOpOp1LopsUS.put(OpOp1.COS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.COS);
- HopsOpOp1LopsUS.put(OpOp1.TAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.TAN);
- HopsOpOp1LopsUS.put(OpOp1.ASIN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ASIN);
- HopsOpOp1LopsUS.put(OpOp1.ACOS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ACOS);
- HopsOpOp1LopsUS.put(OpOp1.ATAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ATAN);
- HopsOpOp1LopsUS.put(OpOp1.SQRT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.SQRT);
- HopsOpOp1LopsUS.put(OpOp1.EXP, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.EXP);
- HopsOpOp1LopsUS.put(OpOp1.LOG, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.LOG);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_INT);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN);
- HopsOpOp1LopsUS.put(OpOp1.NROW, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NROW);
- HopsOpOp1LopsUS.put(OpOp1.NCOL, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NCOL);
- HopsOpOp1LopsUS.put(OpOp1.LENGTH, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.LENGTH);
- HopsOpOp1LopsUS.put(OpOp1.PRINT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.PRINT);
- HopsOpOp1LopsUS.put(OpOp1.ROUND, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ROUND);
- HopsOpOp1LopsUS.put(OpOp1.CEIL, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CEIL);
- HopsOpOp1LopsUS.put(OpOp1.FLOOR, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.FLOOR);
- HopsOpOp1LopsUS.put(OpOp1.STOP, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.STOP);
- }
-
- protected static final HashMap<Hop.OpOp1, String> HopsOpOp12String;
- static {
- HopsOpOp12String = new HashMap<OpOp1, String>();
- HopsOpOp12String.put(OpOp1.ABS, "abs");
- HopsOpOp12String.put(OpOp1.CAST_AS_SCALAR, "castAsScalar");
- HopsOpOp12String.put(OpOp1.COS, "cos");
- HopsOpOp12String.put(OpOp1.EIGEN, "eigen");
- HopsOpOp12String.put(OpOp1.EXP, "exp");
- HopsOpOp12String.put(OpOp1.IQM, "iqm");
- HopsOpOp12String.put(OpOp1.MEDIAN, "median");
- HopsOpOp12String.put(OpOp1.LENGTH, "length");
- HopsOpOp12String.put(OpOp1.LOG, "log");
- HopsOpOp12String.put(OpOp1.NCOL, "ncol");
- HopsOpOp12String.put(OpOp1.NOT, "!");
- HopsOpOp12String.put(OpOp1.NROW, "nrow");
- HopsOpOp12String.put(OpOp1.PRINT, "print");
- HopsOpOp12String.put(OpOp1.ROUND, "round");
- HopsOpOp12String.put(OpOp1.SIN, "sin");
- HopsOpOp12String.put(OpOp1.SQRT, "sqrt");
- HopsOpOp12String.put(OpOp1.TAN, "tan");
- HopsOpOp12String.put(OpOp1.ASIN, "asin");
- HopsOpOp12String.put(OpOp1.ACOS, "acos");
- HopsOpOp12String.put(OpOp1.ATAN, "atan");
- HopsOpOp12String.put(OpOp1.STOP, "stop");
- HopsOpOp12String.put(OpOp1.INVERSE, "inv");
- HopsOpOp12String.put(OpOp1.SPROP, "sprop");
- HopsOpOp12String.put(OpOp1.SIGMOID, "sigmoid");
- }
-
- protected static final HashMap<Hop.ParamBuiltinOp, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes> HopsParameterizedBuiltinLops;
- static {
- HopsParameterizedBuiltinLops = new HashMap<Hop.ParamBuiltinOp, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes>();
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.CDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.INVCDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORM, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM);
- }
-
- protected static final HashMap<Hop.OpOp2, String> HopsOpOp2String;
- static {
- HopsOpOp2String = new HashMap<Hop.OpOp2, String>();
- HopsOpOp2String.put(OpOp2.PLUS, "+");
- HopsOpOp2String.put(OpOp2.MINUS, "-");
- HopsOpOp2String.put(OpOp2.MINUS_NZ, "-nz");
- HopsOpOp2String.put(OpOp2.MINUS1_MULT, "-1*");
- HopsOpOp2String.put(OpOp2.MULT, "*");
- HopsOpOp2String.put(OpOp2.DIV, "/");
- HopsOpOp2String.put(OpOp2.MODULUS, "%%");
- HopsOpOp2String.put(OpOp2.INTDIV, "%/%");
- HopsOpOp2String.put(OpOp2.MIN, "min");
- HopsOpOp2String.put(OpOp2.MAX, "max");
- HopsOpOp2String.put(OpOp2.LESSEQUAL, "<=");
- HopsOpOp2String.put(OpOp2.LESS, "<");
- HopsOpOp2String.put(OpOp2.GREATEREQUAL, ">=");
- HopsOpOp2String.put(OpOp2.GREATER, ">");
- HopsOpOp2String.put(OpOp2.EQUAL, "=");
- HopsOpOp2String.put(OpOp2.NOTEQUAL, "!=");
- HopsOpOp2String.put(OpOp2.OR, "|");
- HopsOpOp2String.put(OpOp2.AND, "&");
- HopsOpOp2String.put(OpOp2.LOG, "log");
- HopsOpOp2String.put(OpOp2.LOG_NZ, "log_nz");
- HopsOpOp2String.put(OpOp2.POW, "^");
- HopsOpOp2String.put(OpOp2.CONCAT, "concat");
- HopsOpOp2String.put(OpOp2.INVALID, "?");
- HopsOpOp2String.put(OpOp2.QUANTILE, "quantile");
- HopsOpOp2String.put(OpOp2.INTERQUANTILE, "interquantile");
- HopsOpOp2String.put(OpOp2.IQM, "IQM");
- HopsOpOp2String.put(OpOp2.MEDIAN, "median");
- HopsOpOp2String.put(OpOp2.CENTRALMOMENT, "cm");
- HopsOpOp2String.put(OpOp2.COVARIANCE, "cov");
- HopsOpOp2String.put(OpOp2.CBIND, "cbind");
- HopsOpOp2String.put(OpOp2.RBIND, "rbind");
- HopsOpOp2String.put(OpOp2.SOLVE, "solve");
- HopsOpOp2String.put(OpOp2.SEQINCR, "seqincr");
- }
-
- public static String getOpOp2String( OpOp2 op ) {
- return HopsOpOp2String.get(op);
- }
-
- protected static final HashMap<Hop.OpOp3, String> HopsOpOp3String;
- static {
- HopsOpOp3String = new HashMap<Hop.OpOp3, String>();
- HopsOpOp3String.put(OpOp3.QUANTILE, "quantile");
- HopsOpOp3String.put(OpOp3.INTERQUANTILE, "interquantile");
- HopsOpOp3String.put(OpOp3.CTABLE, "ctable");
- HopsOpOp3String.put(OpOp3.CENTRALMOMENT, "cm");
- HopsOpOp3String.put(OpOp3.COVARIANCE, "cov");
- }
-
- protected static final HashMap<Hop.OpOp4, String> HopsOpOp4String;
- static {
- HopsOpOp4String = new HashMap<Hop.OpOp4, String>();
- HopsOpOp4String.put(OpOp4.WSLOSS, "wsloss");
- HopsOpOp4String.put(OpOp4.WSIGMOID, "wsigmoid");
- }
-
- protected static final HashMap<Hop.Direction, String> HopsDirection2String;
- static {
- HopsDirection2String = new HashMap<Hop.Direction, String>();
- HopsDirection2String.put(Direction.RowCol, "RC");
- HopsDirection2String.put(Direction.Col, "C");
- HopsDirection2String.put(Direction.Row, "R");
- }
-
- protected static final HashMap<Hop.AggOp, String> HopsAgg2String;
- static {
- HopsAgg2String = new HashMap<Hop.AggOp, String>();
- HopsAgg2String.put(AggOp.SUM, "+");
- HopsAgg2String.put(AggOp.SUM_SQ, "sq+");
- HopsAgg2String.put(AggOp.PROD, "*");
- HopsAgg2String.put(AggOp.MIN, "min");
- HopsAgg2String.put(AggOp.MAX, "max");
- HopsAgg2String.put(AggOp.MAXINDEX, "maxindex");
- HopsAgg2String.put(AggOp.MININDEX, "minindex");
- HopsAgg2String.put(AggOp.TRACE, "trace");
- HopsAgg2String.put(AggOp.MEAN, "mean");
- }
-
- protected static final HashMap<Hop.ReOrgOp, String> HopsTransf2String;
- static {
- HopsTransf2String = new HashMap<ReOrgOp, String>();
- HopsTransf2String.put(ReOrgOp.TRANSPOSE, "t");
- HopsTransf2String.put(ReOrgOp.DIAG, "diag");
- HopsTransf2String.put(ReOrgOp.RESHAPE, "rshape");
- HopsTransf2String.put(ReOrgOp.SORT, "sort");
- }
-
- protected static final HashMap<DataOpTypes, String> HopsData2String;
- static {
- HopsData2String = new HashMap<Hop.DataOpTypes, String>();
- HopsData2String.put(DataOpTypes.PERSISTENTREAD, "PRead");
- HopsData2String.put(DataOpTypes.PERSISTENTWRITE, "PWrite");
- HopsData2String.put(DataOpTypes.TRANSIENTWRITE, "TWrite");
- HopsData2String.put(DataOpTypes.TRANSIENTREAD, "TRead");
- }
-
- public static boolean isFunction(OpOp2 op)
- {
- return op == OpOp2.MIN || op == OpOp2.MAX ||
- op == OpOp2.LOG;// || op == OpOp2.CONCAT; //concat is || in Netezza
- }
-
- public static boolean isSupported(OpOp2 op)
- {
- return op != OpOp2.INVALID && op != OpOp2.QUANTILE &&
- op != OpOp2.INTERQUANTILE && op != OpOp2.IQM;
- }
-
- public static boolean isFunction(OpOp1 op)
- {
- return op == OpOp1.SIN || op == OpOp1.TAN || op == OpOp1.COS ||
- op == OpOp1.ABS || op == OpOp1.EXP || op == OpOp1.LOG ||
- op == OpOp1.ROUND || op == OpOp1.SQRT;
- }
-
- public static boolean isBooleanOperation(OpOp2 op)
- {
- return op == OpOp2.AND || op == OpOp2.EQUAL ||
- op == OpOp2.GREATER || op == OpOp2.GREATEREQUAL ||
- op == OpOp2.LESS || op == OpOp2.LESSEQUAL ||
- op == OpOp2.OR;
- }
-
- /**
- *
- * @param op
- * @return
- */
- public static OpOp2 getOpOp2ForOuterVectorOperation(String op)
- {
- if( "+".equals(op) ) return OpOp2.PLUS;
- else if( "-".equals(op) ) return OpOp2.MINUS;
- else if( "*".equals(op) ) return OpOp2.MULT;
- else if( "/".equals(op) ) return OpOp2.DIV;
- else if( "%%".equals(op) ) return OpOp2.MODULUS;
- else if( "%/%".equals(op) ) return OpOp2.INTDIV;
- else if( "min".equals(op) ) return OpOp2.MIN;
- else if( "max".equals(op) ) return OpOp2.MAX;
- else if( "<=".equals(op) ) return OpOp2.LESSEQUAL;
- else if( "<".equals(op) ) return OpOp2.LESS;
- else if( ">=".equals(op) ) return OpOp2.GREATEREQUAL;
- else if( ">".equals(op) ) return OpOp2.GREATER;
- else if( "==".equals(op) ) return OpOp2.EQUAL;
- else if( "!=".equals(op) ) return OpOp2.NOTEQUAL;
- else if( "|".equals(op) ) return OpOp2.OR;
- else if( "&".equals(op) ) return OpOp2.AND;
- else if( "log".equals(op) ) return OpOp2.LOG;
- else if( "^".equals(op) ) return OpOp2.POW;
-
- return null;
- }
-
- public static ValueType getResultValueType(ValueType vt1, ValueType vt2)
- {
- if(vt1 == ValueType.STRING || vt2 == ValueType.STRING)
- return ValueType.STRING;
- else if(vt1 == ValueType.DOUBLE || vt2 == ValueType.DOUBLE)
- return ValueType.DOUBLE;
- else
- return ValueType.INT;
- }
-
- /////////////////////////////////////
- // methods for dynamic re-compilation
- /////////////////////////////////////
-
- /**
- * Indicates if dynamic recompilation is required for this hop.
- */
- public boolean requiresRecompile()
- {
- return _requiresRecompile;
- }
-
- public void setRequiresRecompile()
- {
- _requiresRecompile = true;
- }
-
- public void unsetRequiresRecompile()
- {
- _requiresRecompile = false;
- }
-
- /**
- * Update the output size information for this hop.
- */
- public abstract void refreshSizeInformation();
-
- /**
- * Util function for refreshing scalar rows input parameter.
- */
- protected void refreshRowsParameterInformation( Hop input )
- {
- long size = computeSizeInformation(input);
-
- //always set the computed size not just if known (positive) in order to allow
- //recompile with unknowns to reset sizes (otherwise potential for incorrect results)
- setDim1( size );
- }
-
-
- /**
- * Util function for refreshing scalar cols input parameter.
- */
- protected void refreshColsParameterInformation( Hop input )
- {
- long size = computeSizeInformation(input);
-
- //always set the computed size not just if known (positive) in order to allow
- //recompile with unknowns to reset sizes (otherwise potential for incorrect results)
- setDim2( size );
- }
-
- /**
- *
- * @param input
- * @return
- */
- public long computeSizeInformation( Hop input )
- {
- long ret = -1;
-
- try
- {
- long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap<Long,Long>());
- if( tmp!=Long.MAX_VALUE )
- ret = tmp;
- }
- catch(Exception ex)
- {
- LOG.error("Failed to compute size information.", ex);
- ret = -1;
- }
-
- return ret;
- }
-
- /**
- *
- * @param input
- * @param vars
- */
- public void refreshRowsParameterInformation( Hop input, LocalVariableMap vars )
- {
- long size = computeSizeInformation(input, vars);
-
- //always set the computed size not just if known (positive) in order to allow
- //recompile with unknowns to reset sizes (otherwise potential for incorrect results)
- setDim1( size );
- }
-
- /**
- *
- * @param input
- * @param vars
- */
- public void refreshColsParameterInformation( Hop input, LocalVariableMap vars )
- {
- long size = computeSizeInformation(input, vars);
-
- //always set the computed size not just if known (positive) in order to allow
- //recompile with unknowns to reset sizes (otherwise potential for incorrect results)
- setDim2( size );
- }
-
- /**
- *
- * @param input
- * @param vars
- * @return
- */
- public long computeSizeInformation( Hop input, LocalVariableMap vars )
- {
- long ret = -1;
-
- try
- {
- long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap<Long,Long>(), vars);
- if( tmp!=Long.MAX_VALUE )
- ret = tmp;
- }
- catch(Exception ex)
- {
- LOG.error("Failed to compute size information.", ex);
- ret = -1;
- }
-
- return ret;
- }
-
- /**
- *
- * @param input
- * @return
- */
- public double computeBoundsInformation( Hop input )
- {
- double ret = Double.MAX_VALUE;
-
- try
- {
- ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap<Long, Double>());
- }
- catch(Exception ex)
- {
- LOG.error("Failed to compute bounds information.", ex);
- ret = Double.MAX_VALUE;
- }
-
- return ret;
- }
-
- /**
- * Computes bound information for sequence if possible, otherwise returns
- * Double.MAX_VALUE
- *
- * @param input
- * @param vars
- * @return
- */
- public double computeBoundsInformation( Hop input, LocalVariableMap vars )
- {
- double ret = Double.MAX_VALUE;
-
- try
- {
- ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap<Long, Double>(), vars);
-
- }
- catch(Exception ex)
- {
- LOG.error("Failed to compute bounds information.", ex);
- ret = Double.MAX_VALUE;
- }
-
- return ret;
- }
-
-
- /**
- * Compute worst case estimate for size expression based on worst-case
- * statistics of inputs. Limited set of supported operations in comparison
- * to refresh rows/cols.
- *
- * @param input
- * @param memo
- */
- protected long computeDimParameterInformation( Hop input, MemoTable memo )
- {
- long ret = -1;
-
- if( input instanceof UnaryOp )
- {
- if( ((UnaryOp)input).getOp() == Hop.OpOp1.NROW )
- {
- MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0));
- if( mc.getRows()>0 )
- ret = mc.getRows();
- }
- else if ( ((UnaryOp)input).getOp() == Hop.OpOp1.NCOL )
- {
- MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0));
- if( mc.getCols()>0 )
- ret = mc.getCols();
- }
- }
- else if ( input instanceof LiteralOp )
- {
- ret = UtilFunctions.parseToLong(input.getName());
- }
- else if ( input instanceof BinaryOp )
- {
- long dim = rEvalSimpleBinaryLongExpression(input, new HashMap<Long, Long>(), memo);
- if( dim != Long.MAX_VALUE ) //if known
- ret = dim ;
- }
-
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @return
- */
- protected long rEvalSimpleBinaryLongExpression( Hop root, HashMap<Long, Long> valMemo, MemoTable memo )
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- long ret = Long.MAX_VALUE;
-
- if( root instanceof LiteralOp )
- {
- long dim = UtilFunctions.parseToLong(root.getName());
- if( dim != -1 ) //if known
- ret = dim;
- }
- else if( root instanceof UnaryOp )
- {
- UnaryOp uroot = (UnaryOp) root;
- long dim = -1;
- if(uroot.getOp() == Hop.OpOp1.NROW)
- {
- MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0));
- dim = mc.getRows();
- }
- else if( uroot.getOp() == Hop.OpOp1.NCOL )
- {
- MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0));
- dim = mc.getCols();
- }
- if( dim != -1 ) //if known
- ret = dim;
- }
- else if( root instanceof BinaryOp )
- {
- if( OptimizerUtils.ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION )
- {
- BinaryOp broot = (BinaryOp) root;
- long lret = rEvalSimpleBinaryLongExpression(broot.getInput().get(0), valMemo, memo);
- long rret = rEvalSimpleBinaryLongExpression(broot.getInput().get(1), valMemo, memo);
- //note: positive and negative values might be valid subexpressions
- if( lret!=Long.MAX_VALUE && rret!=Long.MAX_VALUE ) //if known
- {
- switch( broot.getOp() )
- {
- case PLUS: ret = lret + rret; break;
- case MULT: ret = lret * rret; break;
- case MIN: ret = Math.min(lret, rret); break;
- case MAX: ret = Math.max(lret, rret); break;
- default: ret = Long.MAX_VALUE;
- }
- }
- //exploit min constraint to propagate
- else if( broot.getOp()==OpOp2.MIN && (lret!=Double.MAX_VALUE || rret!=Double.MAX_VALUE) )
- {
- ret = Math.min(lret, rret);
- }
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
-
- /**
- *
- * @return
- */
- public String constructBaseDir()
- {
- StringBuilder sb = new StringBuilder();
- sb.append( ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE) );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( Lop.PROCESS_PREFIX );
- sb.append( DMLScript.getUUID() );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( ProgramConverter.CP_ROOT_THREAD_ID );
- sb.append( Lop.FILE_SEPARATOR );
-
- return sb.toString();
- }
-
- /**
- * Clones the attributes of that and copies it over to this.
- *
- * @param that
- * @throws HopsException
- */
- protected void clone( Hop that, boolean withRefs )
- throws CloneNotSupportedException
- {
- if( withRefs )
- throw new CloneNotSupportedException( "Hops deep copy w/ lops/inputs/parents not supported." );
-
- _ID = that._ID;
- _name = that._name;
- _dataType = that._dataType;
- _valueType = that._valueType;
- _visited = that._visited;
- _dim1 = that._dim1;
- _dim2 = that._dim2;
- _rows_in_block = that._rows_in_block;
- _cols_in_block = that._cols_in_block;
- _nnz = that._nnz;
-
- //no copy of lops (regenerated)
- _parent = new ArrayList<Hop>();
- _input = new ArrayList<Hop>();
- _lops = null;
-
- _etype = that._etype;
- _etypeForced = that._etypeForced;
- _outputMemEstimate = that._outputMemEstimate;
- _memEstimate = that._memEstimate;
- _processingMemEstimate = that._processingMemEstimate;
- _requiresRecompile = that._requiresRecompile;
- _requiresReblock = that._requiresReblock;
- _requiresCheckpoint = that._requiresCheckpoint;
- _outputEmptyBlocks = that._outputEmptyBlocks;
-
- _beginLine = that._beginLine;
- _beginColumn = that._beginColumn;
- _endLine = that._endLine;
- _endColumn = that._endColumn;
- }
-
- public abstract Object clone() throws CloneNotSupportedException;
-
- public abstract boolean compare( Hop that );
-
-
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for Hops
- ///////////////////////////////////////////////////////////////////////////
- public int _beginLine, _beginColumn;
- public int _endLine, _endColumn;
-
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(int blp, int bcp, int elp, int ecp){
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- /**
- * Sets the linenumbers of this hop to a given lop.
- *
- * @param lop
- */
- protected void setLineNumbers(Lop lop)
- {
- lop.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- }
-
-} // end class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/HopsException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/HopsException.java b/src/main/java/com/ibm/bi/dml/hops/HopsException.java
deleted file mode 100644
index d1c4901..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/HopsException.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * <p>Exception occurring in the HOP level.</p>
- */
-public class HopsException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- /**
- * @see java.lang.Exception#Exception()
- */
- public HopsException() {
- super();
- }
-
- /**
- * @see java.lang.Exception#Exception(String)
- */
- public HopsException(String message) {
- super(message);
- }
-
- /**
- * @see java.lang.Exception#Exception(Throwable)
- */
- public HopsException(Throwable cause) {
- super(cause);
- }
-
- /**
- * @see java.lang.Exception#Exception(String, Throwable)
- */
- public HopsException(String message, Throwable cause) {
- super(message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/IndexingOp.java b/src/main/java/com/ibm/bi/dml/hops/IndexingOp.java
deleted file mode 100644
index b351d61..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/IndexingOp.java
+++ /dev/null
@@ -1,464 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RangeBasedReIndex;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-//for now only works for range based indexing op
-public class IndexingOp extends Hop
-{
- public static String OPSTRING = "rix"; //"Indexing";
-
- private boolean _rowLowerEqualsUpper = false;
- private boolean _colLowerEqualsUpper = false;
-
- private enum IndexingMethod {
- CP_RIX, //in-memory range index
- MR_RIX, //general case range reindex
- MR_VRIX, //vector (row/col) range index
- };
-
-
- private IndexingOp() {
- //default constructor for clone
- }
-
- //right indexing doesn't really need the dimensionality of the left matrix
- //private static Lops dummy=new Data(null, Data.OperationTypes.READ, null, "-1", DataType.SCALAR, ValueType.INT, false);
- public IndexingOp(String l, DataType dt, ValueType vt, Hop inpMatrix, Hop inpRowL, Hop inpRowU, Hop inpColL, Hop inpColU, boolean passedRowsLEU, boolean passedColsLEU) {
- super(l, dt, vt);
-
- getInput().add(0, inpMatrix);
- getInput().add(1, inpRowL);
- getInput().add(2, inpRowU);
- getInput().add(3, inpColL);
- getInput().add(4, inpColU);
-
- // create hops if one of them is null
- inpMatrix.getParent().add(this);
- inpRowL.getParent().add(this);
- inpRowU.getParent().add(this);
- inpColL.getParent().add(this);
- inpColU.getParent().add(this);
-
- // set information whether left indexing operation involves row (n x 1) or column (1 x m) matrix
- setRowLowerEqualsUpper(passedRowsLEU);
- setColLowerEqualsUpper(passedColsLEU);
- }
-
-
- public boolean getRowLowerEqualsUpper(){
- return _rowLowerEqualsUpper;
- }
-
- public boolean getColLowerEqualsUpper() {
- return _colLowerEqualsUpper;
- }
-
- public void setRowLowerEqualsUpper(boolean passed){
- _rowLowerEqualsUpper = passed;
- }
-
- public void setColLowerEqualsUpper(boolean passed) {
- _colLowerEqualsUpper = passed;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- Hop input = getInput().get(0);
-
- //rewrite remove unnecessary right indexing
- if( dimsKnown() && input.dimsKnown()
- && getDim1() == input.getDim1() && getDim2() == input.getDim2() )
- {
- setLops( input.constructLops() );
- }
- //actual lop construction, incl operator selection
- else
- {
- try {
- ExecType et = optFindExecType();
- if(et == ExecType.MR) {
- IndexingMethod method = optFindIndexingMethod( _rowLowerEqualsUpper, _colLowerEqualsUpper,
- input._dim1, input._dim2, _dim1, _dim2);
-
- Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
- input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
- getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
- getDataType(), getValueType(), et);
-
- setOutputDimensions(reindex);
- setLineNumbers(reindex);
-
- if( method == IndexingMethod.MR_RIX )
- {
- Group group1 = new Group( reindex, Group.OperationTypes.Sort,
- DataType.MATRIX, getValueType());
- setOutputDimensions(group1);
- setLineNumbers(group1);
-
- Aggregate agg1 = new Aggregate(
- group1, Aggregate.OperationTypes.Sum, DataType.MATRIX,
- getValueType(), et);
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
-
- setLops(agg1);
- }
- else //method == IndexingMethod.MR_VRIX
- {
- setLops(reindex);
- }
- }
- else if( et == ExecType.SPARK )
- {
- IndexingMethod method = optFindIndexingMethod( _rowLowerEqualsUpper, _colLowerEqualsUpper,
- input._dim1, input._dim2, _dim1, _dim2);
- SparkAggType aggtype = (method==IndexingMethod.MR_VRIX) ?
- SparkAggType.NONE : SparkAggType.MULTI_BLOCK;
-
- Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
- input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
- getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
- getDataType(), getValueType(), aggtype, et);
-
- setOutputDimensions(reindex);
- setLineNumbers(reindex);
- setLops(reindex);
- }
- else //CP
- {
- Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
- input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
- getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
- getDataType(), getValueType(), et);
-
- setOutputDimensions(reindex);
- setLineNumbers(reindex);
- setLops(reindex);
- }
- } catch (Exception e) {
- throw new HopsException(this.printErrorLocation() + "In IndexingOp Hop, error constructing Lops " , e);
- }
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += OPSTRING;
- return s;
- }
-
- public void printMe() throws HopsException {
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- public void computeMemEstimate( MemoTable memo )
- {
- //default behavior
- super.computeMemEstimate(memo);
-
- //try to infer via worstcase input statistics (for the case of dims known
- //but nnz initially unknown)
- MatrixCharacteristics mcM1 = memo.getAllInputStats(getInput().get(0));
- if( dimsKnown() && mcM1.getNonZeros()>=0 ){
- long lnnz = mcM1.getNonZeros(); //worst-case output nnz
- double lOutMemEst = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- if( lOutMemEst<_outputMemEstimate ){
- _outputMemEstimate = lOutMemEst;
- _memEstimate = getInputOutputSize();
- }
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- Hop input = getInput().get(0); //original matrix
- MatrixCharacteristics mc = memo.getAllInputStats(input);
- if( mc != null )
- {
- long lnnz = mc.dimsKnown()?Math.min(mc.getRows()*mc.getCols(), mc.getNonZeros()):-1;
- //worst-case is input size, but dense
- ret = new long[]{mc.getRows(), mc.getCols(), lnnz};
-
- //exploit column/row indexing information
- if( _rowLowerEqualsUpper ) ret[0]=1;
- if( _colLowerEqualsUpper ) ret[1]=1;
-
- //infer tight block indexing size
- Hop rl = getInput().get(1);
- Hop ru = getInput().get(2);
- Hop cl = getInput().get(3);
- Hop cu = getInput().get(4);
- if( isBlockIndexingExpression(rl, ru) )
- ret[0] = getBlockIndexingExpressionSize(rl, ru);
- if( isBlockIndexingExpression(cl, cu) )
- ret[1] = getBlockIndexingExpressionSize(cl, cu);
- }
-
- return ret;
- }
-
- /**
- * Indicates if the lbound:rbound expressions is of the form
- * "(c * (i - 1) + 1) : (c * i)", where we could use c as a tight size estimate.
- *
- * @param lbound
- * @param ubound
- * @return
- */
- private boolean isBlockIndexingExpression(Hop lbound, Hop ubound)
- {
- boolean ret = false;
- LiteralOp constant = null;
- DataOp var = null;
-
- //handle lower bound
- if( lbound instanceof BinaryOp && ((BinaryOp)lbound).getOp()==OpOp2.PLUS
- && lbound.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)lbound.getInput().get(1))==1
- && lbound.getInput().get(0) instanceof BinaryOp)
- {
- BinaryOp lmult = (BinaryOp)lbound.getInput().get(0);
- if( lmult.getOp()==OpOp2.MULT && lmult.getInput().get(0) instanceof LiteralOp
- && lmult.getInput().get(1) instanceof BinaryOp )
- {
- BinaryOp lminus = (BinaryOp)lmult.getInput().get(1);
- if( lminus.getOp()==OpOp2.MINUS && lminus.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)lminus.getInput().get(1))==1
- && lminus.getInput().get(0) instanceof DataOp )
- {
- constant = (LiteralOp)lmult.getInput().get(0);
- var = (DataOp) lminus.getInput().get(0);
- }
- }
- }
-
- //handle upper bound
- if( var != null && constant != null && ubound instanceof BinaryOp
- && ubound.getInput().get(0) instanceof LiteralOp
- && ubound.getInput().get(1) instanceof DataOp
- && ubound.getInput().get(1).getName().equals(var.getName()) )
- {
- LiteralOp constant2 = (LiteralOp)ubound.getInput().get(0);
- ret = ( HopRewriteUtils.getDoubleValueSafe(constant) ==
- HopRewriteUtils.getDoubleValueSafe(constant2) );
- }
-
- return ret;
- }
-
- /**
- *
- * @param lbound
- * @param ubound
- * @return
- */
- private long getBlockIndexingExpressionSize(Hop lbound, Hop ubound)
- {
- //NOTE: ensure consistency with isBlockIndexingExpression
- LiteralOp c = (LiteralOp) ubound.getInput().get(0); //(c*i)
- return HopRewriteUtils.getIntValueSafe(c);
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if ( getInput().get(0).areDimsBelowThreshold() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- return _etype;
- }
-
- /**
- *
- * @param singleRow
- * @param singleCol
- * @param m1_dim1
- * @param m1_dim2
- * @param m2_dim1
- * @param m2_dim2
- * @return
- */
- private static IndexingMethod optFindIndexingMethod( boolean singleRow, boolean singleCol, long m1_dim1, long m1_dim2, long m2_dim1, long m2_dim2 )
- {
- if( singleRow && m1_dim2 == m2_dim2 && m2_dim2!=-1
- || singleCol && m1_dim1 == m2_dim1 && m2_dim1!=-1 )
- {
- return IndexingMethod.MR_VRIX;
- }
-
- return IndexingMethod.MR_RIX; //general case
- }
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = getInput().get(0); //original matrix
- Hop input2 = getInput().get(1); //inpRowL
- Hop input3 = getInput().get(2); //inpRowU
- Hop input4 = getInput().get(3); //inpColL
- Hop input5 = getInput().get(4); //inpColU
-
- //parse input information
- boolean allRows =
- ( input2 instanceof LiteralOp && HopRewriteUtils.getIntValueSafe((LiteralOp)input2)==1
- && input3 instanceof UnaryOp && ((UnaryOp)input3).getOp() == OpOp1.NROW );
- boolean allCols =
- ( input4 instanceof LiteralOp && HopRewriteUtils.getIntValueSafe((LiteralOp)input4)==1
- && input5 instanceof UnaryOp && ((UnaryOp)input5).getOp() == OpOp1.NCOL );
- boolean constRowRange = (input2 instanceof LiteralOp && input3 instanceof LiteralOp);
- boolean constColRange = (input4 instanceof LiteralOp && input5 instanceof LiteralOp);
-
- //set dimension information
- if( _rowLowerEqualsUpper ) //ROWS
- setDim1(1);
- else if( allRows )
- setDim1(input1.getDim1());
- else if( constRowRange ){
- setDim1( HopRewriteUtils.getIntValueSafe((LiteralOp)input3)
- -HopRewriteUtils.getIntValueSafe((LiteralOp)input2)+1 );
- }
- else if( isBlockIndexingExpression(input2, input3) ) {
- setDim1(getBlockIndexingExpressionSize(input2, input3));
- }
-
- if( _colLowerEqualsUpper ) //COLS
- setDim2(1);
- else if( allCols )
- setDim2(input1.getDim2());
- else if( constColRange ){
- setDim2( HopRewriteUtils.getIntValueSafe((LiteralOp)input5)
- -HopRewriteUtils.getIntValueSafe((LiteralOp)input4)+1 );
- }
- else if( isBlockIndexingExpression(input4, input5) ) {
- setDim2(getBlockIndexingExpressionSize(input4, input5));
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- IndexingOp ret = new IndexingOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof IndexingOp)
- || getInput().size() != that.getInput().size() )
- {
- return false;
- }
-
- return ( getInput().get(0) == that.getInput().get(0)
- && getInput().get(1) == that.getInput().get(1)
- && getInput().get(2) == that.getInput().get(2)
- && getInput().get(3) == that.getInput().get(3)
- && getInput().get(4) == that.getInput().get(4));
- }
-}
[65/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
index 8291c45..21c4abe 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.List;
@@ -25,22 +25,22 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.PickByCount.OperationTypes;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.PickByCount.OperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class QuantilePickSPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantileSortSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantileSortSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantileSortSPInstruction.java
index 0b877c3..5109935 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantileSortSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantileSortSPInstruction.java
@@ -15,26 +15,26 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDSortUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDSortUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public class QuantileSortSPInstruction extends UnarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
index ac8dce8..1af7967 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.io.Serializable;
@@ -30,33 +30,33 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
index 372a26c..08c13b0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.io.Serializable;
import java.util.ArrayList;
@@ -32,30 +32,30 @@ import org.apache.spark.util.random.SamplingUtils;
import scala.Tuple2;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixDatagen;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.RandomMatrixGenerator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.Statistics;
public class RandSPInstruction extends UnarySPInstruction
{
@@ -311,8 +311,8 @@ public class RandSPInstruction extends UnarySPInstruction
if ( this.method == DataGenMethod.RAND )
{
// The implementation is in same spirit as MapReduce
- // We generate seeds similar to com.ibm.bi.dml.runtime.matrix.DataGenMR
- // and then generate blocks similar to com.ibm.bi.dml.runtime.matrix.mapred.DataGenMapper
+ // We generate seeds similar to org.apache.sysml.runtime.matrix.DataGenMR
+ // and then generate blocks similar to org.apache.sysml.runtime.matrix.mapred.DataGenMapper
//generate pseudo-random seed (because not specified)
long lSeed = seed; //seed per invocation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
index 079e236..904454c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
@@ -15,32 +15,32 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDProperties;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
+import org.apache.sysml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class ReblockSPInstruction extends UnarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/RelationalBinarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RelationalBinarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RelationalBinarySPInstruction.java
index 2997400..257acab 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RelationalBinarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RelationalBinarySPInstruction.java
@@ -1,14 +1,14 @@
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class RelationalBinarySPInstruction extends BinarySPInstruction {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java
index f69f8c0..2cda50f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -25,28 +25,28 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SortIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.FilterDiagBlocksFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.IsBlockInRange;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDSortUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ReorgMapFunction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.SortIndex;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.functions.FilterDiagBlocksFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.IsBlockInRange;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDSortUtils;
+import org.apache.sysml.runtime.instructions.spark.functions.ReorgMapFunction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ReorgSPInstruction extends UnarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java
index b917158..199f5ce 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.LinkedList;
@@ -26,22 +26,22 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TripleIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TripleIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
index d61bad7..06da8a6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.SPInstructionParser;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.SPInstructionParser;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
index 60fa40c..6f074dd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -26,30 +26,30 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.Ternary;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.CTable;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.Ternary;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.CTable;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class TernarySPInstruction extends ComputationSPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/TsmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TsmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TsmmSPInstruction.java
index d56cd47..a3b3ffa 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TsmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TsmmSPInstruction.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
index 90b4348..e3944b8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.Arrays;
@@ -27,33 +27,33 @@ import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.IndexFunction;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.AggregateDropCorrectionFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixOuterAgg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.UAggOuterChain;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.IndexFunction;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.AggregateDropCorrectionFunction;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixOuterAgg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* Two types of broadcast variables used -- 1. Array of type double. 2.PartitionedMatrixBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
index f543c47..5c1102a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.Not;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.Not;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public abstract class UnarySPInstruction extends ComputationSPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
index 1225cb1..83ef316 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,26 +26,26 @@ import org.apache.spark.Accumulator;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ComputeBinaryBlockNnzFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ConvertMatrixBlockToIJVLines;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.functions.ComputeBinaryBlockNnzFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.ConvertMatrixBlockToIJVLines;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriteSPInstruction extends SPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ZipmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ZipmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ZipmmSPInstruction.java
index 3f51301..d95a395 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ZipmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ZipmmSPInstruction.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class ZipmmSPInstruction extends BinarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java
index a5e4177..60760ad 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import org.apache.spark.Partitioner;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Default partitioner used for all binary block rdd operations in order
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
index a24001a..2d306a7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import org.apache.spark.broadcast.Broadcast;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CorrMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CorrMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CorrMatrixBlock.java
index 1c389d6..5ef0f5b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CorrMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CorrMatrixBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -26,9 +26,9 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataInputStream;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
public class CorrMatrixBlock implements Externalizable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
index ab088fb..da257ec 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LazyIterableIterator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LazyIterableIterator.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LazyIterableIterator.java
index 91affec..2d12922 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LazyIterableIterator.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LazyIterableIterator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java
index fe3306d..3730517 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.util.ArrayList;
import java.util.List;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
public abstract class LineageObject
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
index 0842e9b..205ab80 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.Serializable;
import org.apache.spark.broadcast.Broadcast;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* This class is a wrapper around an array of broadcasts of partitioned matrix blocks,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
index 07f2248..7bd2f6c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -29,16 +29,16 @@ import java.util.ArrayList;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataInputStream;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataOutputStream;
-import com.ibm.bi.dml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.util.IndexRange;
/**
* The main purpose of this class is to provide a handle for partitioned matrix blocks, to be used
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDObject.java
index 84e426f..12a4f84 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDObject.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import org.apache.spark.api.java.JavaPairRDD;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
index 6cf4a6c..259516b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RDDProperties.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
/**
* Class which allows the MLContext to pass properties for the RDD
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RowMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RowMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RowMatrixBlock.java
index 73ddcc7..cee71de 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RowMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/RowMatrixBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -26,9 +26,9 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataInputStream;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
public class RowMatrixBlock implements Externalizable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerLongWritable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerLongWritable.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerLongWritable.java
index 4b47a7e..eb1a608 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerLongWritable.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerLongWritable.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerText.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerText.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerText.java
index 2cb245d..8197bb7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerText.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/SerText.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.data;
+package org.apache.sysml.runtime.instructions.spark.data;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/AggregateDropCorrectionFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/AggregateDropCorrectionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/AggregateDropCorrectionFunction.java
index 09ffdc7..255440d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/AggregateDropCorrectionFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/AggregateDropCorrectionFunction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
public class AggregateDropCorrectionFunction implements Function<MatrixBlock, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ComputeBinaryBlockNnzFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ComputeBinaryBlockNnzFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ComputeBinaryBlockNnzFunction.java
index d125333..6345f9f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ComputeBinaryBlockNnzFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ComputeBinaryBlockNnzFunction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.Accumulator;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
index a51d120..392ab59 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
import java.util.List;
@@ -24,10 +24,10 @@ import org.apache.spark.api.java.JavaRDD;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ConvertColumnRDDToBinaryBlock {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
index 7ba2ee3..5eeb0bc 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.mllib.linalg.DenseMatrix;
@@ -23,9 +23,9 @@ import org.apache.spark.mllib.linalg.SparseMatrix;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ConvertMLLibBlocksToBinaryBlocks implements PairFunction<Tuple2<Tuple2<Object,Object>,Matrix>, MatrixIndexes, MatrixBlock> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMatrixBlockToIJVLines.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMatrixBlockToIJVLines.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMatrixBlockToIJVLines.java
index e321ee6..e6224a8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMatrixBlockToIJVLines.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMatrixBlockToIJVLines.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.Iterator;
@@ -22,9 +22,9 @@ import org.apache.spark.api.java.function.FlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryBlockToTextCellConverter;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.BinaryBlockToTextCellConverter;
public class ConvertMatrixBlockToIJVLines implements FlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, String> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java
index 342573e..4c71441 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
index 1ab013e..3d00f87 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.data.TextToBinaryCellConverter;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.TextToBinaryCellConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ConvertTextLineToBinaryCellFunction implements PairFunction<String, MatrixIndexes, MatrixCell> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
index 6984e32..3572b31 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.function.Function;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java
index 764cdf7..797e306 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java
@@ -14,12 +14,12 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import scala.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockFunction.java
index 5e820f2..9528026 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockFunction.java
@@ -14,10 +14,10 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* General purpose copy function for binary block rdds. This function can be used in
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
index 9682817..5eeac76 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
@@ -14,14 +14,14 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* General purpose copy function for binary block rdds. This function can be used in
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java
index b510a6f..ce37aaa 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
index ec41e48..fb8490d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
@@ -23,11 +23,11 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
index 7a76df6..f15c9b3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
@@ -23,12 +23,12 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ExtractGroup implements PairFlatMapFunction<Tuple2<MatrixIndexes,Tuple2<MatrixBlock, MatrixBlock>>, Long, WeightedCell> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java
index e93e51a..53eb772 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
@@ -23,12 +23,12 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ExtractGroupNWeights implements PairFlatMapFunction<Tuple2<MatrixIndexes,Tuple2<Iterable<Tuple2<MatrixBlock,MatrixBlock>>,Iterable<MatrixBlock>>>, Long, WeightedCell> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterDiagBlocksFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterDiagBlocksFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterDiagBlocksFunction.java
index 54574b6..76981b4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterDiagBlocksFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterDiagBlocksFunction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterNonEmptyBlocksFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterNonEmptyBlocksFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterNonEmptyBlocksFunction.java
index c92ce96..9a6fab9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterNonEmptyBlocksFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FilterNonEmptyBlocksFunction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
index d586e98..c1d8d92 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Function class find the tuple that matches the given MatrixIndexes.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java
index 9547044..5eb8a74 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.Row;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
public class GetMIMBFromRow implements PairFunction<Row, MatrixIndexes, MatrixBlock> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLBlock.java
index 5ee570b..4adf270 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.io.Serializable;
@@ -24,9 +24,9 @@ import org.apache.spark.sql.Row;
import scala.Tuple2;
-import com.ibm.bi.dml.api.MLBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.api.MLBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
public class GetMLBlock implements Function<Tuple2<MatrixIndexes,MatrixBlock>, Row>, Serializable {
[61/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
index 505dba7..beeac58 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,20 +29,20 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PMMJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
@SuppressWarnings("rawtypes")
public abstract class MapperBase extends MRBaseForCommonInstructions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
index 4cd9520..22ec070 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
@@ -25,13 +25,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class PartialAggregator extends MMCJMRCache
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
index 1adddb5..ba4a9fd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,14 +26,14 @@ import java.util.Comparator;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.OutputCollector;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.AdaptivePartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.PartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedAdaptivePartialBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.AdaptivePartialBlock;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.PartialBlock;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.matrix.data.TaggedAdaptivePartialBlock;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockMapper.java
index d7514f5..c659fec 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,16 +29,16 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.AdaptivePartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.PartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedAdaptivePartialBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.AdaptivePartialBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.PartialBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedAdaptivePartialBlock;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockReducer.java
index 375283e..0601550 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.HashMap;
@@ -27,14 +27,14 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.AdaptivePartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.PartialBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedAdaptivePartialBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.AdaptivePartialBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.PartialBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedAdaptivePartialBlock;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java
index 13c4a63..0f034c9 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,18 +28,18 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.mr.AggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.TernaryInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class ReduceBase extends MRBaseForCommonInstructions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateBinaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateBinaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateBinaryOperator.java
index ada01ce..a27965a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateBinaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateBinaryOperator.java
@@ -16,13 +16,13 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class AggregateBinaryOperator extends Operator implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateOperator.java
index a87070d..7d6227a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateOperator.java
@@ -16,17 +16,17 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
import java.io.Serializable;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlusSq;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.Or;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.Or;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class AggregateOperator extends Operator implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateUnaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateUnaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateUnaryOperator.java
index a249a42..36b02df 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateUnaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/AggregateUnaryOperator.java
@@ -16,14 +16,14 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.functionobjects.IndexFunction;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlusSq;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.Or;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.IndexFunction;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.Or;
+import org.apache.sysml.runtime.functionobjects.Plus;
public class AggregateUnaryOperator extends Operator
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
index af1b7a3..5d2d304 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
@@ -16,30 +16,30 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
import java.io.Serializable;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.runtime.functionobjects.And;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Divide;
-import com.ibm.bi.dml.runtime.functionobjects.Equals;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.IntegerDivide;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.MinusNz;
-import com.ibm.bi.dml.runtime.functionobjects.Modulus;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.NotEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Or;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.Power;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.runtime.functionobjects.And;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Divide;
+import org.apache.sysml.runtime.functionobjects.Equals;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.IntegerDivide;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.MinusNz;
+import org.apache.sysml.runtime.functionobjects.Modulus;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.NotEquals;
+import org.apache.sysml.runtime.functionobjects.Or;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.Power;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
public class BinaryOperator extends Operator implements Serializable
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/CMOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/CMOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/CMOperator.java
index 7a834a7..eabcf1b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/CMOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/CMOperator.java
@@ -16,10 +16,10 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class CMOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/COVOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/COVOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/COVOperator.java
index ab55968..6eb6988 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/COVOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/COVOperator.java
@@ -16,9 +16,9 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.functionobjects.COV;
public class COVOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/LeftScalarOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/LeftScalarOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/LeftScalarOperator.java
index 05aee6f..890d1a7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/LeftScalarOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/LeftScalarOperator.java
@@ -16,15 +16,15 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Power;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.Power;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class LeftScalarOperator extends ScalarOperator
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/Operator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/Operator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/Operator.java
index f64f75d..f113c16 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/Operator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/Operator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/QuaternaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/QuaternaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/QuaternaryOperator.java
index c0b1412..a3ec42b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/QuaternaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/QuaternaryOperator.java
@@ -16,17 +16,17 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply2;
-import com.ibm.bi.dml.runtime.functionobjects.Power2;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Multiply2;
+import org.apache.sysml.runtime.functionobjects.Power2;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class QuaternaryOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReIndexOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/ReIndexOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReIndexOperator.java
index d1c8f3d..0c196b6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/ReIndexOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReIndexOperator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
public class ReIndexOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReorgOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/ReorgOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReorgOperator.java
index cdb7263..7315e85 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/ReorgOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/ReorgOperator.java
@@ -16,11 +16,11 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.functionobjects.IndexFunction;
+import org.apache.sysml.runtime.functionobjects.IndexFunction;
public class ReorgOperator extends Operator implements Serializable
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/RightScalarOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/RightScalarOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/RightScalarOperator.java
index 282b526..706f273 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/RightScalarOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/RightScalarOperator.java
@@ -16,14 +16,14 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+package org.apache.sysml.runtime.matrix.operators;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class RightScalarOperator extends ScalarOperator
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
index 8608a50..5b86f53 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
@@ -16,23 +16,23 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.And;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
-import com.ibm.bi.dml.runtime.functionobjects.Equals;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.MinusNz;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply2;
-import com.ibm.bi.dml.runtime.functionobjects.NotEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Power;
-import com.ibm.bi.dml.runtime.functionobjects.Power2;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.And;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
+import org.apache.sysml.runtime.functionobjects.Equals;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.MinusNz;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Multiply2;
+import org.apache.sysml.runtime.functionobjects.NotEquals;
+import org.apache.sysml.runtime.functionobjects.Power;
+import org.apache.sysml.runtime.functionobjects.Power2;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class ScalarOperator extends Operator
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/SimpleOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/SimpleOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/SimpleOperator.java
index d242e43..667e125 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/SimpleOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/SimpleOperator.java
@@ -16,9 +16,9 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.functionobjects.FunctionObject;
+import org.apache.sysml.runtime.functionobjects.FunctionObject;
/*
* Simple operator is just a wrapper for a single function object of any type.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/UnaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/UnaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/UnaryOperator.java
index d9afcf9..508e6d6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/UnaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/UnaryOperator.java
@@ -16,10 +16,10 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
public class UnaryOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/operators/ZeroOutOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/ZeroOutOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/ZeroOutOperator.java
index d05f13e..86118cd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/ZeroOutOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/ZeroOutOperator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.operators;
+package org.apache.sysml.runtime.matrix.operators;
public class ZeroOutOperator extends Operator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
index 4d90654..8fd98b5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.ArrayList;
@@ -37,9 +37,9 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.Pair;
public class CompactDoubleIntInputFormat extends FileInputFormat<MatrixIndexes, MatrixCell>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactInputFormat.java
index a7ab648..87937b4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactInputFormat.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
index f592080..cd2ddcd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.DataOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparable.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparable.java
index 32ecc3d..454ee92 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparable.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparable.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparableDesc.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparableDesc.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparableDesc.java
index 3ff98df..098546a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparableDesc.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortComparableDesc.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import org.apache.hadoop.io.DoubleWritable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortMapper.java
index d7cb257..408d48f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
@@ -26,10 +26,10 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class IndexSortMapper extends MapReduceBase
implements Mapper<MatrixIndexes, MatrixBlock, IndexSortComparable, LongWritable>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortReducer.java
index a4b2a98..e07fd2e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.Iterator;
@@ -27,11 +27,11 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class IndexSortReducer extends MapReduceBase
implements Reducer<IndexSortComparable, LongWritable, MatrixIndexes, MatrixBlock>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupMapper.java
index 344dc12..40a65f9 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.StringTokenizer;
@@ -26,10 +26,10 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class IndexSortStitchupMapper extends MapReduceBase
implements Mapper<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixBlock>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupReducer.java
index e91bf4c..89f7654 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/IndexSortStitchupReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.Iterator;
@@ -26,10 +26,10 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class IndexSortStitchupReducer extends MapReduceBase
implements Reducer<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixBlock>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
index f8c5429..a8ef0eb 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.ArrayList;
@@ -39,10 +39,10 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.Pair;
//key class to read has to be DoubleWritable
public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, MatrixCell>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/ReadWithZeros.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/ReadWithZeros.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/ReadWithZeros.java
index 6553acc..5d0bcce 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/ReadWithZeros.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/ReadWithZeros.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.EOFException;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/SamplingSortMRInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/SamplingSortMRInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/SamplingSortMRInputFormat.java
index 88f91a0..6da873b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/SamplingSortMRInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/SamplingSortMRInputFormat.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.ArrayList;
@@ -37,10 +37,10 @@ import org.apache.hadoop.mapred.SequenceFileRecordReader;
import org.apache.hadoop.util.IndexedSortable;
import org.apache.hadoop.util.QuickSort;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java
index 261d9ce..dae9f3d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
@@ -28,13 +28,13 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineUnaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.mr.CombineUnaryInstruction;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
@SuppressWarnings("rawtypes")
public class ValueSortMapper<KIN extends WritableComparable, VIN extends Writable, KOUT extends WritableComparable, VOUT extends Writable> extends MapReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortReducer.java
index 3bf9035..e2f9577 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.sort;
+package org.apache.sysml.runtime.matrix.sort;
import java.io.IOException;
import java.util.Iterator;
@@ -29,8 +29,8 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.matrix.SortMR;
+import org.apache.sysml.runtime.util.MapReduceTool;
@SuppressWarnings("rawtypes")
public class ValueSortReducer<K extends WritableComparable, V extends Writable> extends MapReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
index 24e6701..cf8434f 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
@@ -16,7 +16,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.util.HashSet;
import org.apache.hadoop.filecache.DistributedCache;
@@ -26,23 +26,23 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.BlockRow;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.WriteCSVMR;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.instructions.InstructionParser;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.BlockRow;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.WriteCSVMR;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CSVReblockReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* MapReduce job that performs the actual data transformations, such as recoding
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMapper.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMapper.java
index f4095a0..c49dc72 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -33,14 +33,14 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockMapper.IndexedBlockRow;
-import com.ibm.bi.dml.runtime.matrix.mapred.MapperBase;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CSVReblockMapper;
+import org.apache.sysml.runtime.matrix.mapred.CSVReblockMapper.IndexedBlockRow;
+import org.apache.sysml.runtime.matrix.mapred.MapperBase;
@SuppressWarnings("deprecation")
public class ApplyTfBBMapper extends MapperBase implements Mapper<LongWritable, Text, TaggedFirstSecondIndexes, CSVReblockMR.BlockRow>{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMR.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMR.java
index b073f09..1441f9c 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -34,11 +34,11 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMapper.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMapper.java
index e26efaf..d2bfc32 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
@@ -31,7 +31,7 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLRuntimeException;
public class ApplyTfCSVMapper implements Mapper<LongWritable, Text, NullWritable, Text> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVSPARK.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVSPARK.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVSPARK.java
index 312d181..0323733 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVSPARK.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfCSVSPARK.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -34,9 +34,9 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
public class ApplyTfCSVSPARK {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
index 10917c0..876dd05 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/BinAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -38,8 +38,8 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.transform.MVImputeAgent.MVMethod;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.transform.MVImputeAgent.MVMethod;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class BinAgent extends TransformationAgent {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
index 9479b2f..999f279 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -44,40 +44,40 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.CSVReBlock;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.AssignRowIDMRReturn;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.transform.TransformationAgent.TX_METHOD;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.JSONHelper;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.CSVReBlock;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.AssignRowIDMRReturn;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.transform.TransformationAgent.TX_METHOD;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.JSONHelper;
public class DataTransform {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/DistinctValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DistinctValue.java b/src/main/java/org/apache/sysml/runtime/transform/DistinctValue.java
index 86b8e3c..bcdd8b0 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DistinctValue.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DistinctValue.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.DataInput;
import java.io.DataOutput;
@@ -29,8 +29,8 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class DistinctValue implements Writable, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
index e776729..d419f31 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -37,7 +37,7 @@ import org.apache.wink.json4j.JSONObject;
import com.google.common.base.Functions;
import com.google.common.collect.Ordering;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class DummycodeAgent extends TransformationAgent {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/GTFMTDMapper.java b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDMapper.java
index 1ebb78c..bf040f3 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/GTFMTDMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
@@ -27,7 +27,7 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
public class GTFMTDMapper implements Mapper<LongWritable, Text, IntWritable, DistinctValue>{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/GTFMTDReducer.java b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDReducer.java
index 83a7b32..b412a72 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/GTFMTDReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/GTFMTDReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -34,8 +34,8 @@ import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class GTFMTDReducer implements Reducer<IntWritable, DistinctValue, Text, LongWritable> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdMR.java b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdMR.java
index c634d40..b6bd081 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdMR.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
@@ -32,8 +32,8 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
/**
* MR Job to Generate Transform Metadata based on a given transformation specification file (JSON format).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdSPARK.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdSPARK.java b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdSPARK.java
index f424cd7..7c34be6 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdSPARK.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/GenTfMtdSPARK.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -37,9 +37,9 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
public class GenTfMtdSPARK {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
index aeec9d6..dc63833 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/MVImputeAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -40,15 +40,15 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.Mean;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.Mean;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class MVImputeAgent extends TransformationAgent {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
index cfda9a3..67abb04 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/OmitAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.util.Arrays;
@@ -30,7 +30,7 @@ import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONException;
import org.apache.wink.json4j.JSONObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class OmitAgent extends TransformationAgent {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
index c4221fc..226250d 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -40,8 +40,8 @@ import org.apache.wink.json4j.JSONObject;
import scala.Tuple2;
import com.google.common.collect.Ordering;
-import com.ibm.bi.dml.runtime.transform.MVImputeAgent.MVMethod;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.transform.MVImputeAgent.MVMethod;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class RecodeAgent extends TransformationAgent {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
index e6e3c72..b49b34c 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.BufferedReader;
import java.io.EOFException;
@@ -37,17 +37,17 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.wink.json4j.JSONException;
import org.apache.wink.json4j.JSONObject;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.JSONHelper;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.JSONHelper;
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/transform/TransformationAgent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/TransformationAgent.java b/src/main/java/org/apache/sysml/runtime/transform/TransformationAgent.java
index 37cbba7..372a0a7 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/TransformationAgent.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/TransformationAgent.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.transform;
+package org.apache.sysml.runtime.transform;
import java.io.IOException;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
index 04eae4f..55afc0b 100644
--- a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.IOException;
@@ -26,8 +26,8 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Custom binary block input format to return the custom record reader.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockRecordReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockRecordReader.java b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockRecordReader.java
index bf2e0ab..7b007d0 100644
--- a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockRecordReader.java
+++ b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockRecordReader.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.IOException;
@@ -23,8 +23,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.SequenceFileRecordReader;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Custom record reader for binary block. Currently its only purpose is to allow for
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index c9d4d91..039b7bf 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.IOException;
import java.util.ArrayList;
@@ -25,24 +25,24 @@ import java.util.Map.Entry;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.io.MatrixWriterFactory;
-import com.ibm.bi.dml.runtime.io.ReadProperties;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.udf.Matrix;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.io.MatrixWriter;
+import org.apache.sysml.runtime.io.MatrixWriterFactory;
+import org.apache.sysml.runtime.io.ReadProperties;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.udf.Matrix;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
index 5f23ace..2b79301 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.DataInput;
import java.io.EOFException;
@@ -23,8 +23,8 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataInput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
index be574ce..7c9a50e 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.util;
+package org.apache.sysml.runtime.util;
import java.io.DataOutput;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataOutput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.matrix.data.MatrixBlockDataOutput;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
/**
* This buffered output stream is essentially a merged version of
[52/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
index 340ab59..dd82e31 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -27,40 +27,40 @@ import org.junit.runners.Suite;
@Suite.SuiteClasses({
// .applications.dml package
- com.ibm.bi.dml.test.integration.applications.dml.ApplyTransformDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.ArimaDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.CsplineCGDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.CsplineDSDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.GLMDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.GNMFDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.HITSDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.ID3DMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.L2SVMDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.LinearLogRegDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.LinearRegressionDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.MDABivariateStatsDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.MultiClassSVMDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.NaiveBayesDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.PageRankDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.dml.WelchTDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.ApplyTransformDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.ArimaDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.CsplineCGDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.CsplineDSDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.GLMDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.GNMFDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.HITSDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.ID3DMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.L2SVMDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.LinearLogRegDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.LinearRegressionDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.MDABivariateStatsDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.MultiClassSVMDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.NaiveBayesDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.PageRankDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.WelchTDMLTest.class,
// .applications.pydml package
- com.ibm.bi.dml.test.integration.applications.pydml.ApplyTransformPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.ArimaPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.CsplineCGPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.CsplineDSPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.GLMPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.GNMFPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.HITSPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.ID3PyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.L2SVMPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.LinearLogRegPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.LinearRegressionPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.MDABivariateStatsPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.MultiClassSVMPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.NaiveBayesPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.PageRankPyDMLTest.class,
- com.ibm.bi.dml.test.integration.applications.pydml.WelchTPyDMLTest.class
+ org.apache.sysml.test.integration.applications.pydml.ApplyTransformPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.ArimaPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.CsplineCGPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.CsplineDSPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.GLMPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.GNMFPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.HITSPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.ID3PyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.L2SVMPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.LinearLogRegPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.LinearRegressionPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.MDABivariateStatsPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.MultiClassSVMPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.NaiveBayesPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.PageRankPyDMLTest.class,
+ org.apache.sysml.test.integration.applications.pydml.WelchTPyDMLTest.class
})
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/applications/descriptivestats/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/applications/descriptivestats/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/applications/descriptivestats/ZPackageSuite.java
index 45fb087..246e14e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/applications/descriptivestats/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/applications/descriptivestats/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/applications/parfor/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/applications/parfor/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/applications/parfor/ZPackageSuite.java
index c3ccf88..5312fad 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/applications/parfor/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/applications/parfor/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
index 3604b06..de7c199 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
index 4d44c81..230f20a 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.append;
+package org.apache.sysml.test.integration.functions.append;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix/ZPackageSuite.java
index afed722..3de02b1 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix;
+package org.apache.sysml.test.integration.functions.binary.matrix;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/ZPackageSuite.java
index f52be69..df8ac12 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_cellwise/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_cellwise;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_cellwise;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/ZPackageSuite.java
index 652c5dc..c04dbc6 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/scalar/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/scalar/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/scalar/ZPackageSuite.java
index a0e21cd..7226ebe 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/scalar/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/binary/scalar/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/blocks/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/blocks/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/blocks/ZPackageSuite.java
index 7c7ac2a..8a6a99c 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/blocks/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/blocks/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.blocks;
+package org.apache.sysml.test.integration.functions.blocks;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/caching/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/caching/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/caching/ZPackageSuite.java
index c2678df..056118a 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/caching/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/caching/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.caching;
+package org.apache.sysml.test.integration.functions.caching;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/data/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/data/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/data/ZPackageSuite.java
index 5b22c3b..fb3515e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/data/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/data/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/dmlscript/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/dmlscript/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/dmlscript/ZPackageSuite.java
index 7c2de0e..4769b18 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/dmlscript/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/dmlscript/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.dmlscript;
+package org.apache.sysml.test.integration.functions.dmlscript;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java
index 3968cdb..e2ecaf1 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.external;
+package org.apache.sysml.test.integration.functions.external;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java
index 2d8eaaf..fc71922 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/indexing/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/indexing/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/indexing/ZPackageSuite.java
index 50b929c..6e95c73 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/indexing/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/indexing/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/ZPackageSuite.java
index 52a7dca..2140297 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/binary/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/binary/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/binary/ZPackageSuite.java
index 8d43f71..d863083 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/binary/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/binary/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.binary;
+package org.apache.sysml.test.integration.functions.io.binary;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/csv/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/csv/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/csv/ZPackageSuite.java
index dfcc423..6ac01ea 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/csv/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/csv/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.csv;
+package org.apache.sysml.test.integration.functions.io.csv;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ZPackageSuite.java
index 8841b56..1db5241 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io.matrixmarket;
+package org.apache.sysml.test.integration.functions.io.matrixmarket;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
index f20917b..f6bae6b 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.jmlc;
+package org.apache.sysml.test.integration.functions.jmlc;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
index a2d033c..39b1a9c 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.misc;
+package org.apache.sysml.test.integration.functions.misc;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/mlcontext/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/mlcontext/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/mlcontext/ZPackageSuite.java
index 8ed2f83..d61b224 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/mlcontext/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/mlcontext/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.mlcontext;
+package org.apache.sysml.test.integration.functions.mlcontext;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/parfor/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/parfor/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/parfor/ZPackageSuite.java
index e8399f5..42768e0 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/parfor/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/parfor/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.parfor;
+package org.apache.sysml.test.integration.functions.parfor;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/piggybacking/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/piggybacking/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/piggybacking/ZPackageSuite.java
index 0ab1d39..035c0b9 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/piggybacking/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/piggybacking/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.piggybacking;
+package org.apache.sysml.test.integration.functions.piggybacking;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/quaternary/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/quaternary/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/quaternary/ZPackageSuite.java
index 1d5782e..5d0a534 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/quaternary/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/quaternary/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.quaternary;
+package org.apache.sysml.test.integration.functions.quaternary;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
index 96c635a..ff87a63 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.recompile;
+package org.apache.sysml.test.integration.functions.recompile;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java
index e81fe96..9bee446 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.reorg;
+package org.apache.sysml.test.integration.functions.reorg;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/terms/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/terms/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/terms/ZPackageSuite.java
index 7f49941..f9b97c8 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/terms/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/terms/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.terms;
+package org.apache.sysml.test.integration.functions.terms;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/ternary/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/ternary/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/ternary/ZPackageSuite.java
index d56f1e1..037beef 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/ternary/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/ternary/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.ternary;
+package org.apache.sysml.test.integration.functions.ternary;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
index bfc6aef..53c2aca 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.transform;
+package org.apache.sysml.test.integration.functions.transform;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
index 7664b29..7a73755 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/scalar/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/scalar/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/scalar/ZPackageSuite.java
index 5a2a8db..be2f45e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/scalar/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/scalar/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
index 9446f89..2868e0e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.vect;
+package org.apache.sysml.test.integration.functions.vect;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
index bd57dd1..ec5e23e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.scalability;
+package org.apache.sysml.test.integration.scalability;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
[45/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/AggBinaryOp.java b/src/main/java/com/ibm/bi/dml/hops/AggBinaryOp.java
deleted file mode 100644
index 005744f..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/AggBinaryOp.java
+++ /dev/null
@@ -1,1959 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.MMCJ;
-import com.ibm.bi.dml.lops.MMRJ;
-import com.ibm.bi.dml.lops.MMTSJ;
-import com.ibm.bi.dml.lops.MMCJ.MMCJType;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.lops.MMZip;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.lops.PMapMult;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.Transform.OperationTypes;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
-
-
-/* Aggregate binary (cell operations): Sum (aij + bij)
- * Properties:
- * Inner Symbol: *, -, +, ...
- * Outer Symbol: +, min, max, ...
- * 2 Operands
- *
- * Semantic: generate indices, align, cross-operate, generate indices, align, aggregate
- */
-
-public class AggBinaryOp extends Hop implements MultiThreadedHop
-{
- public static final double MAPMULT_MEM_MULTIPLIER = 1.0;
- public static MMultMethod FORCED_MMULT_METHOD = null;
-
- public enum MMultMethod {
- CPMM, //cross-product matrix multiplication (mr)
- RMM, //replication matrix multiplication (mr)
- MAPMM_L, //map-side matrix-matrix multiplication using distributed cache (mr/sp)
- MAPMM_R, //map-side matrix-matrix multiplication using distributed cache (mr/sp)
- MAPMM_CHAIN, //map-side matrix-matrix-matrix multiplication using distributed cache, for right input (cp/mr/sp)
- PMAPMM, //partitioned map-side matrix-matrix multiplication (sp)
- PMM, //permutation matrix multiplication using distributed cache, for left input (mr/cp)
- TSMM, //transpose-self matrix multiplication (cp/mr/sp)
- ZIPMM, //zip matrix multiplication (sp)
- MM //in-memory matrix multiplication (cp)
- };
-
- public enum SparkAggType{
- NONE,
- SINGLE_BLOCK,
- MULTI_BLOCK,
- }
-
- private OpOp2 innerOp;
- private AggOp outerOp;
-
- private MMultMethod _method = null;
-
- //hints set by previous to operator selection
- private boolean _hasLeftPMInput = false; //left input is permutation matrix
- private int _maxNumThreads = -1; //-1 for unlimited
-
- private AggBinaryOp() {
- //default constructor for clone
- }
-
- public AggBinaryOp(String l, DataType dt, ValueType vt, OpOp2 innOp,
- AggOp outOp, Hop in1, Hop in2) {
- super(l, dt, vt);
- innerOp = innOp;
- outerOp = outOp;
- getInput().add(0, in1);
- getInput().add(1, in2);
- in1.getParent().add(this);
- in2.getParent().add(this);
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- public void setHasLeftPMInput(boolean flag) {
- _hasLeftPMInput = flag;
- }
-
- public boolean hasLeftPMInput(){
- return _hasLeftPMInput;
- }
-
- @Override
- public void setMaxNumThreads( int k ) {
- _maxNumThreads = k;
- }
-
- @Override
- public int getMaxNumThreads() {
- return _maxNumThreads;
- }
-
- public MMultMethod getMMultMethod(){
- return _method;
- }
-
- /**
- * NOTE: overestimated mem in case of transpose-identity matmult, but 3/2 at worst
- * and existing mem estimate advantageous in terms of consistency hops/lops,
- * and some special cases internally materialize the transpose for better cache locality
- */
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- //construct matrix mult lops (currently only supported aggbinary)
- if ( isMatrixMultiply() )
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
-
- //matrix mult operation selection part 1 (CP vs MR vs Spark)
- ExecType et = optFindExecType();
-
- //matrix mult operation selection part 2 (specific pattern)
- MMTSJType mmtsj = checkTransposeSelf(); //determine tsmm pattern
- ChainType chain = checkMapMultChain(); //determine mmchain pattern
-
- if( et == ExecType.CP )
- {
- //matrix mult operation selection part 3 (CP type)
- _method = optFindMMultMethodCP ( input1.getDim1(), input1.getDim2(),
- input2.getDim1(), input2.getDim2(), mmtsj, chain, _hasLeftPMInput );
-
- //dispatch CP lops construction
- switch( _method ){
- case TSMM:
- constructCPLopsTSMM( mmtsj );
- break;
- case MAPMM_CHAIN:
- constructCPLopsMMChain( chain );
- break;
- case PMM:
- constructCPLopsPMM();
- break;
- case MM:
- constructCPLopsMM();
- break;
- default:
- throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing CP lops.");
- }
- }
- else if( et == ExecType.SPARK )
- {
- //matrix mult operation selection part 3 (SPARK type)
- boolean tmmRewrite = input1 instanceof ReorgOp && ((ReorgOp)input1).getOp()==ReOrgOp.TRANSPOSE;
- _method = optFindMMultMethodSpark (
- input1.getDim1(), input1.getDim2(), input1.getRowsInBlock(), input1.getColsInBlock(), input1.getNnz(),
- input2.getDim1(), input2.getDim2(), input2.getRowsInBlock(), input2.getColsInBlock(), input2.getNnz(),
- mmtsj, chain, _hasLeftPMInput, tmmRewrite );
-
- //dispatch SPARK lops construction
- switch( _method )
- {
- case TSMM:
- constructSparkLopsTSMM( mmtsj );
- break;
- case MAPMM_L:
- case MAPMM_R:
- constructSparkLopsMapMM( _method );
- break;
- case MAPMM_CHAIN:
- constructSparkLopsMapMMChain( chain );
- break;
- case PMAPMM:
- constructSparkLopsPMapMM();
- break;
- case CPMM:
- constructSparkLopsCPMM();
- break;
- case RMM:
- constructSparkLopsRMM();
- break;
- case PMM:
- constructSparkLopsPMM();
- break;
- case ZIPMM:
- constructSparkLopsZIPMM();
- break;
-
- default:
- throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing SPARK lops.");
- }
- }
- else if( et == ExecType.MR )
- {
- //matrix mult operation selection part 3 (MR type)
- _method = optFindMMultMethodMR (
- input1.getDim1(), input1.getDim2(), input1.getRowsInBlock(), input1.getColsInBlock(), input1.getNnz(),
- input2.getDim1(), input2.getDim2(), input2.getRowsInBlock(), input2.getColsInBlock(), input2.getNnz(),
- mmtsj, chain, _hasLeftPMInput);
-
- //dispatch MR lops construction
- switch( _method ) {
- case MAPMM_L:
- case MAPMM_R:
- constructMRLopsMapMM( _method );
- break;
- case MAPMM_CHAIN:
- constructMRLopsMapMMChain( chain );
- break;
- case CPMM:
- constructMRLopsCPMM();
- break;
- case RMM:
- constructMRLopsRMM();
- break;
- case TSMM:
- constructMRLopsTSMM( mmtsj );
- break;
- case PMM:
- constructMRLopsPMM();
- break;
- default:
- throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing MR lops.");
- }
- }
- }
- else
- throw new HopsException(this.printErrorLocation() + "Invalid operation in AggBinary Hop, aggBin(" + innerOp + "," + outerOp + ") while constructing lops.");
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- public String getOpString() {
- //ba - binary aggregate, for consistency with runtime
- String s = "ba(" +
- HopsAgg2String.get(outerOp) +
- HopsOpOp2String.get(innerOp)+")";
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" InnerOperation: " + innerOp);
- LOG.debug(" OuterOperation: " + outerOp);
- for (Hop h : getInput()) {
- h.printMe();
- }
- ;
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
-
-
- @Override
- public void computeMemEstimate(MemoTable memo)
- {
- //extension of default compute memory estimate in order to
- //account for smaller tsmm memory requirements.
- super.computeMemEstimate(memo);
-
- //tsmm left is guaranteed to require only X but not t(X), while
- //tsmm right might have additional requirements to transpose X if sparse
- //NOTE: as a heuristic this correction is only applied if not a column vector because
- //most other vector operations require memory for at least two vectors (we aim for
- //consistency in order to prevent anomalies in parfor opt leading to small degree of par)
- MMTSJType mmtsj = checkTransposeSelf();
- if( mmtsj.isLeft() && getInput().get(1).dimsKnown() && getInput().get(1).getDim2()>1 ) {
- _memEstimate = _memEstimate - getInput().get(0)._outputMemEstimate;
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- //NOTES:
- // * The estimate for transpose-self is the same as for normal matrix multiplications
- // because (1) this decouples the decision of TSMM over default MM and (2) some cases
- // of TSMM internally materialize the transpose for efficiency.
- // * All matrix multiplications internally use dense output representations for efficiency.
- // This is reflected in our conservative memory estimate. However, we additionally need
- // to account for potential final dense/sparse transformations via processing mem estimates.
- double sparsity = 1.0;
- /*
- if( isMatrixMultiply() ) {
- if( nnz < 0 ){
- Hops input1 = getInput().get(0);
- Hops input2 = getInput().get(1);
- if( input1.dimsKnown() && input2.dimsKnown() )
- {
- double sp1 = (input1.getNnz()>0) ? OptimizerUtils.getSparsity(input1.getDim1(), input1.getDim2(), input1.getNnz()) : 1.0;
- double sp2 = (input2.getNnz()>0) ? OptimizerUtils.getSparsity(input2.getDim1(), input2.getDim2(), input2.getNnz()) : 1.0;
- sparsity = OptimizerUtils.getMatMultSparsity(sp1, sp2, input1.getDim1(), input1.getDim2(), input2.getDim2(), true);
- }
- }
- else //sparsity known (e.g., inferred from worst case estimates)
- sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- }
- */
- //currently always estimated as dense in order to account for dense intermediate without unnecessary overestimation
- double ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- //account for potential final dense-sparse transformation (worst-case sparse representation)
- if( dim2 >= 2 ) //vectors always dense
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, MatrixBlock.SPARSITY_TURN_POINT);
-
- return ret;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- MatrixCharacteristics[] mc = memo.getAllInputStats(getInput());
- if( mc[0].rowsKnown() && mc[1].colsKnown() ) {
- ret = new long[3];
- ret[0] = mc[0].getRows();
- ret[1] = mc[1].getCols();
- double sp1 = (mc[0].getNonZeros()>0) ? OptimizerUtils.getSparsity(mc[0].getRows(), mc[0].getCols(), mc[0].getNonZeros()) : 1.0;
- double sp2 = (mc[1].getNonZeros()>0) ? OptimizerUtils.getSparsity(mc[1].getRows(), mc[1].getCols(), mc[1].getNonZeros()) : 1.0;
- ret[2] = (long) ( ret[0] * ret[1] * OptimizerUtils.getMatMultSparsity(sp1, sp2, ret[0], mc[0].getCols(), ret[1], true));
- }
-
- return ret;
- }
-
-
- public boolean isMatrixMultiply() {
- return ( this.innerOp == OpOp2.MULT && this.outerOp == AggOp.SUM );
- }
-
- private boolean isOuterProduct() {
- if ( getInput().get(0).isVector() && getInput().get(1).isVector() ) {
- if ( getInput().get(0).getDim1() == 1 && getInput().get(0).getDim1() > 1
- && getInput().get(1).getDim1() > 1 && getInput().get(1).getDim2() == 1 )
- return true;
- else
- return false;
- }
- return false;
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = findExecTypeByMemEstimate();
- }
- // choose CP if the dimensions of both inputs are below Hops.CPThreshold
- // OR if it is vector-vector inner product
- else if ( (getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold())
- || (getInput().get(0).isVector() && getInput().get(1).isVector() && !isOuterProduct()) )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //spark-specific decision refinement (execute binary aggregate w/ left spark input and
- //single parent also in spark because it's likely cheap and reduces data transfer)
- if( _etype == ExecType.CP && _etypeForced != ExecType.CP
- && !(getInput().get(0) instanceof ReorgOp && ((ReorgOp)getInput().get(0)).getOp()==ReOrgOp.TRANSPOSE)
- && !(getInput().get(0) instanceof DataOp) //input is not checkpoint
- && getInput().get(0).getParent().size()==1 //bagg is only parent
- && !getInput().get(0).areDimsBelowThreshold()
- && getInput().get(0).optFindExecType() == ExecType.SPARK
- && getInput().get(0).getOutputMemEstimate()>getOutputMemEstimate() )
- {
- //pull unary aggregate into spark
- _etype = ExecType.SPARK;
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
-
- return _etype;
- }
-
- /**
- * TSMM: Determine if XtX pattern applies for this aggbinary and if yes
- * which type.
- *
- * @return
- */
- public MMTSJType checkTransposeSelf()
- {
- MMTSJType ret = MMTSJType.NONE;
-
- Hop in1 = getInput().get(0);
- Hop in2 = getInput().get(1);
-
- if( in1 instanceof ReorgOp
- && ((ReorgOp)in1).getOp() == ReOrgOp.TRANSPOSE
- && in1.getInput().get(0) == in2 )
- {
- ret = MMTSJType.LEFT;
- }
-
- if( in2 instanceof ReorgOp
- && ((ReorgOp)in2).getOp() == ReOrgOp.TRANSPOSE
- && in2.getInput().get(0) == in1 )
- {
- ret = MMTSJType.RIGHT;
- }
-
- return ret;
- }
-
- /**
- * MapMultChain: Determine if XtwXv/XtXv pattern applies for this aggbinary
- * and if yes which type.
- *
- * @return
- */
- public ChainType checkMapMultChain()
- {
- ChainType chainType = ChainType.NONE;
-
- Hop in1 = getInput().get(0);
- Hop in2 = getInput().get(1);
-
- //check for transpose left input (both chain types)
- if( in1 instanceof ReorgOp && ((ReorgOp)in1).getOp() == ReOrgOp.TRANSPOSE )
- {
- Hop X = in1.getInput().get(0);
-
- //check mapmultchain patterns
- //t(X)%*%(w*(X%*%v))
- if( in2 instanceof BinaryOp )
- {
- Hop in3b = in2.getInput().get(1);
- if( in3b instanceof AggBinaryOp )
- {
- Hop in4 = in3b.getInput().get(0);
- if( X == in4 ) //common input
- chainType = ChainType.XtwXv;
- }
- }
- //t(X)%*%(X%*%v)
- else if( in2 instanceof AggBinaryOp )
- {
- Hop in3 = in2.getInput().get(0);
- if( X == in3 ) //common input
- chainType = ChainType.XtXv;
- }
- }
-
- return chainType;
- }
-
- //////////////////////////
- // CP Lops generation
- /////////////////////////
-
- /**
- *
- * @param mmtsj
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsTSMM( MMTSJType mmtsj )
- throws HopsException, LopsException
- {
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- Lop matmultCP = new MMTSJ(getInput().get(mmtsj.isLeft()?1:0).constructLops(),
- getDataType(), getValueType(), ExecType.CP, mmtsj, k);
-
- matmultCP.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers( matmultCP );
- setLops(matmultCP);
- }
-
- /**
- *
- * @param chain
- * @throws LopsException
- * @throws HopsException
- */
- private void constructCPLopsMMChain( ChainType chain )
- throws LopsException, HopsException
- {
- MapMultChain mapmmchain = null;
- if( chain == ChainType.XtXv ) {
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), getDataType(), getValueType(), ExecType.CP);
- }
- else { //if( chainType == ChainType.XtwXv )
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), getDataType(), getValueType(), ExecType.CP);
- }
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- mapmmchain.setNumThreads( k );
-
- //set basic lop properties
- setOutputDimensions(mapmmchain);
- setLineNumbers(mapmmchain);
- setLops(mapmmchain);
- }
-
- /**
- * NOTE: exists for consistency since removeEmtpy might be scheduled to MR
- * but matrix mult on small output might be scheduled to CP. Hence, we
- * need to handle directly passed selection vectors in CP as well.
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsPMM()
- throws HopsException, LopsException
- {
- Hop pmInput = getInput().get(0);
- Hop rightInput = getInput().get(1);
-
- Hop nrow = HopRewriteUtils.createValueHop(pmInput, true); //NROW
- HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0);
- nrow.setForcedExecType(ExecType.CP);
- HopRewriteUtils.copyLineNumbers(this, nrow);
- Lop lnrow = nrow.constructLops();
-
- PMMJ pmm = new PMMJ(pmInput.constructLops(), rightInput.constructLops(), lnrow, getDataType(), getValueType(), false, false, ExecType.CP);
-
- //set degree of parallelism
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- pmm.setNumThreads(k);
-
- pmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pmm);
-
- setLops(pmm);
-
- HopRewriteUtils.removeChildReference(pmInput, nrow);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructCPLopsMM()
- throws HopsException, LopsException
- {
- Lop matmultCP = null;
- if( isLeftTransposeRewriteApplicable(true, false) ) {
- matmultCP = constructCPLopsMMWithLeftTransposeRewrite();
- }
- else {
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- matmultCP = new Binary(getInput().get(0).constructLops(),getInput().get(1).constructLops(),
- Binary.OperationTypes.MATMULT, getDataType(), getValueType(), ExecType.CP, k);
- }
-
- setOutputDimensions(matmultCP);
- setLineNumbers( matmultCP );
- setLops(matmultCP);
- }
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructCPLopsMMWithLeftTransposeRewrite()
- throws HopsException, LopsException
- {
- Hop X = getInput().get(0).getInput().get(0); //guaranteed to exists
- Hop Y = getInput().get(1);
-
- //right vector transpose
- Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(tY);
-
- //matrix mult
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- Lop mult = new Binary(tY, X.constructLops(), Binary.OperationTypes.MATMULT, getDataType(), getValueType(), ExecType.CP, k);
- mult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mult);
-
- //result transpose (dimensions set outside)
- Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
-
- return out;
- }
-
- //////////////////////////
- // Spark Lops generation
- /////////////////////////
-
- /**
- *
- * @param mmtsj
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsTSMM(MMTSJType mmtsj)
- throws HopsException, LopsException
- {
- Hop input = getInput().get(mmtsj.isLeft()?1:0);
- MMTSJ tsmm = new MMTSJ(input.constructLops(), getDataType(), getValueType(), ExecType.SPARK, mmtsj);
- setOutputDimensions(tsmm);
- setLineNumbers(tsmm);
- setLops(tsmm);
- }
-
- /**
- *
- * @param method
- * @throws LopsException
- * @throws HopsException
- */
- private void constructSparkLopsMapMM(MMultMethod method)
- throws LopsException, HopsException
- {
- Lop mapmult = null;
- if( isLeftTransposeRewriteApplicable(false, false) )
- {
- mapmult = constructSparkLopsMapMMWithLeftTransposeRewrite();
- }
- else
- {
- // If number of columns is smaller than block size then explicit aggregation is not required.
- // i.e., entire matrix multiplication can be performed in the mappers.
- boolean needAgg = requiresAggregation(method);
- SparkAggType aggtype = getSparkMMAggregationType(needAgg);
- _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
-
- //core matrix mult
- mapmult = new MapMult( getInput().get(0).constructLops(), getInput().get(1).constructLops(),
- getDataType(), getValueType(), (method==MMultMethod.MAPMM_R), false,
- _outputEmptyBlocks, aggtype);
- }
- setOutputDimensions(mapmult);
- setLineNumbers(mapmult);
- setLops(mapmult);
- }
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructSparkLopsMapMMWithLeftTransposeRewrite()
- throws HopsException, LopsException
- {
- Hop X = getInput().get(0).getInput().get(0); //guaranteed to exists
- Hop Y = getInput().get(1);
-
- //right vector transpose
- Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(tY);
-
- //matrix mult spark
- boolean needAgg = requiresAggregation(MMultMethod.MAPMM_R);
- SparkAggType aggtype = getSparkMMAggregationType(needAgg);
- _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
-
- Lop mult = new MapMult( tY, X.constructLops(), getDataType(), getValueType(),
- false, false, _outputEmptyBlocks, aggtype);
- mult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mult);
-
- //result transpose (dimensions set outside)
- Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
-
- return out;
- }
-
- /**
- *
- * @param chain
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsMapMMChain(ChainType chain)
- throws LopsException, HopsException
- {
- MapMultChain mapmmchain = null;
- if( chain == ChainType.XtXv ) {
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), getDataType(), getValueType(), ExecType.SPARK);
- }
- else { //if( chainType == ChainType.XtwXv )
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), getDataType(), getValueType(), ExecType.SPARK);
- }
- setOutputDimensions(mapmmchain);
- setLineNumbers(mapmmchain);
- setLops(mapmmchain);
- }
-
- /**
- *
- * @throws LopsException
- * @throws HopsException
- */
- private void constructSparkLopsPMapMM()
- throws LopsException, HopsException
- {
- PMapMult pmapmult = new PMapMult(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getDataType(), getValueType() );
- setOutputDimensions(pmapmult);
- setLineNumbers(pmapmult);
- setLops(pmapmult);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsCPMM()
- throws HopsException, LopsException
- {
- if( isLeftTransposeRewriteApplicable(false, false) )
- {
- setLops( constructSparkLopsCPMMWithLeftTransposeRewrite() );
- }
- else
- {
- SparkAggType aggtype = getSparkMMAggregationType(true);
-
- Lop cpmm = new MMCJ(getInput().get(0).constructLops(), getInput().get(1).constructLops(),
- getDataType(), getValueType(), aggtype, ExecType.SPARK);
- setOutputDimensions( cpmm );
- setLineNumbers( cpmm );
- setLops( cpmm );
- }
- }
-
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructSparkLopsCPMMWithLeftTransposeRewrite()
- throws HopsException, LopsException
- {
- SparkAggType aggtype = getSparkMMAggregationType(true);
-
- Hop X = getInput().get(0).getInput().get(0); //guaranteed to exists
- Hop Y = getInput().get(1);
-
- //right vector transpose CP
- Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(tY);
-
- //matrix multiply
- MMCJ mmcj = new MMCJ(tY, X.constructLops(), getDataType(), getValueType(), aggtype, ExecType.SPARK);
- mmcj.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mmcj);
-
- //result transpose CP
- Lop out = new Transform(mmcj, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- return out;
- }
-
- /**
- *
- * @param chain
- * @throws LopsException
- * @throws HopsException
- */
- private void constructSparkLopsRMM()
- throws LopsException, HopsException
- {
- Lop rmm = new MMRJ(getInput().get(0).constructLops(),getInput().get(1).constructLops(),
- getDataType(), getValueType(), ExecType.SPARK);
- setOutputDimensions(rmm);
- setLineNumbers( rmm );
- setLops(rmm);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsPMM()
- throws HopsException, LopsException
- {
- //PMM has two potential modes (a) w/ full permutation matrix input, and
- //(b) w/ already condensed input vector of target row positions.
-
- Hop pmInput = getInput().get(0);
- Hop rightInput = getInput().get(1);
- long brlen = pmInput.getRowsInBlock();
- long bclen = pmInput.getColsInBlock();
-
- //a) full permutation matrix input (potentially without empty block materialized)
- Lop lpmInput = pmInput.constructLops();
- if( pmInput.getDim2() != 1 ) //not a vector
- {
- //compute condensed permutation matrix vector input
- //v = rowMaxIndex(t(pm)) * rowMax(t(pm))
- ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
- transpose.setForcedExecType(ExecType.SPARK);
- HopRewriteUtils.copyLineNumbers(this, transpose);
-
- AggUnaryOp agg1 = new AggUnaryOp("tmp2a", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAXINDEX, Direction.Row, transpose);
- HopRewriteUtils.setOutputBlocksizes(agg1, brlen, bclen);
- agg1.refreshSizeInformation();
- agg1.setForcedExecType(ExecType.SPARK);
- HopRewriteUtils.copyLineNumbers(this, agg1);
-
- AggUnaryOp agg2 = new AggUnaryOp("tmp2b", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, Direction.Row, transpose);
- HopRewriteUtils.setOutputBlocksizes(agg2, brlen, bclen);
- agg2.refreshSizeInformation();
- agg2.setForcedExecType(ExecType.SPARK);
- HopRewriteUtils.copyLineNumbers(this, agg2);
-
- BinaryOp mult = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, agg1, agg2);
- HopRewriteUtils.setOutputBlocksizes(mult, brlen, bclen);
- mult.refreshSizeInformation();
- mult.setForcedExecType(ExecType.SPARK);
- //mult.computeMemEstimate(memo); //select exec type
- HopRewriteUtils.copyLineNumbers(this, mult);
-
- lpmInput = mult.constructLops();
-
- HopRewriteUtils.removeChildReference(pmInput, transpose);
- }
-
- //b) condensed permutation matrix vector input (target rows)
- Hop nrow = HopRewriteUtils.createValueHop(pmInput, true); //NROW
- HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0);
- nrow.setForcedExecType(ExecType.CP);
- HopRewriteUtils.copyLineNumbers(this, nrow);
- Lop lnrow = nrow.constructLops();
-
- _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
- PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), lnrow, getDataType(), getValueType(), false, _outputEmptyBlocks, ExecType.SPARK);
- setOutputDimensions(pmm);
- setLineNumbers(pmm);
- setLops(pmm);
-
- HopRewriteUtils.removeChildReference(pmInput, nrow);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructSparkLopsZIPMM()
- throws HopsException, LopsException
- {
- //zipmm applies to t(X)%*%y if ncol(X)<=blocksize and it prevents
- //unnecessary reshuffling by keeping the original indexes (and partitioning)
- //joining the datasets, and internally doing the necessary transpose operations
-
- Hop left = getInput().get(0).getInput().get(0); //x out of t(X)
- Hop right = getInput().get(1); //y
-
- //determine left-transpose rewrite beneficial
- boolean tRewrite = (left.getDim1()*left.getDim2() >= right.getDim1()*right.getDim2());
-
- Lop zipmm = new MMZip(left.constructLops(), right.constructLops(), getDataType(), getValueType(), tRewrite, ExecType.SPARK);
- setOutputDimensions(zipmm);
- setLineNumbers( zipmm );
- setLops(zipmm);
- }
-
- //////////////////////////
- // MR Lops generation
- /////////////////////////
-
-
- /**
- *
- * @param method
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsMapMM(MMultMethod method)
- throws HopsException, LopsException
- {
- if( method == MMultMethod.MAPMM_R && isLeftTransposeRewriteApplicable(false, true) )
- {
- setLops( constructMRLopsMapMMWithLeftTransposeRewrite() );
- }
- else //GENERAL CASE
- {
- // If number of columns is smaller than block size then explicit aggregation is not required.
- // i.e., entire matrix multiplication can be performed in the mappers.
- boolean needAgg = requiresAggregation(method);
- boolean needPart = requiresPartitioning(method, false);
- _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
-
- //pre partitioning
- Lop leftInput = getInput().get(0).constructLops();
- Lop rightInput = getInput().get(1).constructLops();
- if( needPart ) {
- if( (method==MMultMethod.MAPMM_L) ) //left in distributed cache
- {
- Hop input = getInput().get(0);
- ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz()))
- < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : ExecType.MR; //operator selection
- leftInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
- leftInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
- setLineNumbers(leftInput);
- }
- else //right side in distributed cache
- {
- Hop input = getInput().get(1);
- ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz()))
- < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : ExecType.MR; //operator selection
- rightInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- rightInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
- setLineNumbers(rightInput);
- }
- }
-
- //core matrix mult
- MapMult mapmult = new MapMult( leftInput, rightInput, getDataType(), getValueType(),
- (method==MMultMethod.MAPMM_R), needPart, _outputEmptyBlocks);
- mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mapmult);
-
- //post aggregation
- if (needAgg) {
- Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
-
- grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(agg1);
-
- // aggregation uses kahanSum but the inputs do not have correction values
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
-
- setLops(agg1);
- }
- else {
- setLops(mapmult);
- }
- }
- }
-
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructMRLopsMapMMWithLeftTransposeRewrite()
- throws HopsException, LopsException
- {
- Hop X = getInput().get(0).getInput().get(0); //guaranteed to exists
- Hop Y = getInput().get(1);
-
- //right vector transpose CP
- Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(tY);
-
- //matrix mult
-
- // If number of columns is smaller than block size then explicit aggregation is not required.
- // i.e., entire matrix multiplication can be performed in the mappers.
- boolean needAgg = ( X.getDim1() <= 0 || X.getDim1() > X.getRowsInBlock() );
- boolean needPart = requiresPartitioning(MMultMethod.MAPMM_R, true); //R disregarding transpose rewrite
-
- //pre partitioning
- Lop dcinput = null;
- if( needPart ) {
- ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(Y.getDim2(), Y.getDim1(), OptimizerUtils.getSparsity(Y.getDim2(), Y.getDim1(), Y.getNnz()))
- < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : ExecType.MR; //operator selection
- dcinput = new DataPartition(tY, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
- dcinput.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(dcinput);
- }
- else
- dcinput = tY;
-
- MapMult mapmult = new MapMult(dcinput, X.constructLops(), getDataType(), getValueType(), false, needPart, false);
- mapmult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mapmult);
-
- //post aggregation
- Lop mult = null;
- if( needAgg ) {
- Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
- grp.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- agg1.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(agg1);
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
- mult = agg1;
- }
- else
- mult = mapmult;
-
- //result transpose CP
- Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- return out;
- }
-
-
- /**
- *
- * @param chainType
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsMapMMChain( ChainType chainType )
- throws HopsException, LopsException
- {
- Lop mapmult = null;
-
- if( chainType == ChainType.XtXv )
- {
- //v never needs partitioning because always single block
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1);
-
- //core matrix mult
- mapmult = new MapMultChain( hX.constructLops(), hv.constructLops(), getDataType(), getValueType(), ExecType.MR);
- mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mapmult);
- }
- else //if( chainType == ChainType.XtwXv )
- {
- //v never needs partitioning because always single block
- Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
-
- double mestW = OptimizerUtils.estimateSize(hw.getDim1(), hw.getDim2());
- boolean needPart = !hw.dimsKnown() || hw.getDim1() * hw.getDim2() > DistributedCacheInput.PARTITION_SIZE;
- Lop X = hX.constructLops(), v = hv.constructLops(), w = null;
- if( needPart ){ //requires partitioning
- w = new DataPartition(hw.constructLops(), DataType.MATRIX, ValueType.DOUBLE, (mestW>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- w.getOutputParameters().setDimensions(hw.getDim1(), hw.getDim2(), getRowsInBlock(), getColsInBlock(), hw.getNnz());
- setLineNumbers(w);
- }
- else
- w = hw.constructLops();
-
- //core matrix mult
- mapmult = new MapMultChain( X, v, w, getDataType(), getValueType(), ExecType.MR);
- mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(mapmult);
- }
-
- //post aggregation
- Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
- grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum
- setLineNumbers(agg1);
-
- setLops(agg1);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsCPMM()
- throws HopsException, LopsException
- {
- if( isLeftTransposeRewriteApplicable(false, false) )
- {
- setLops( constructMRLopsCPMMWithLeftTransposeRewrite() );
- }
- else //general case
- {
- Hop X = getInput().get(0);
- Hop Y = getInput().get(1);
-
- MMCJType type = getMMCJAggregationType(X, Y);
- MMCJ mmcj = new MMCJ(X.constructLops(), Y.constructLops(), getDataType(), getValueType(), type, ExecType.MR);
- setOutputDimensions(mmcj);
- setLineNumbers(mmcj);
-
- Group grp = new Group(mmcj, Group.OperationTypes.Sort, getDataType(), getValueType());
- setOutputDimensions(grp);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
-
- // aggregation uses kahanSum but the inputs do not have correction values
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
-
- setLops(agg1);
- }
- }
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructMRLopsCPMMWithLeftTransposeRewrite()
- throws HopsException, LopsException
- {
- Hop X = getInput().get(0).getInput().get(0); //guaranteed to exists
- Hop Y = getInput().get(1);
-
- //right vector transpose CP
- Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
- setLineNumbers(tY);
-
- //matrix multiply
- MMCJType type = getMMCJAggregationType(X, Y);
- MMCJ mmcj = new MMCJ(tY, X.constructLops(), getDataType(), getValueType(), type, ExecType.MR);
- setOutputDimensions(mmcj);
- setLineNumbers(mmcj);
-
- Group grp = new Group(mmcj, Group.OperationTypes.Sort, getDataType(), getValueType());
- setOutputDimensions(grp);
- setLineNumbers(grp);
-
- Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
-
- // aggregation uses kahanSum but the inputs do not have correction values
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
-
-
- //result transpose CP
- Lop out = new Transform(agg1, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
- out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- return out;
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsRMM()
- throws HopsException, LopsException
- {
- MMRJ rmm = new MMRJ(getInput().get(0).constructLops(), getInput().get(1).constructLops(),
- getDataType(), getValueType(), ExecType.MR);
-
- setOutputDimensions(rmm);
- setLineNumbers(rmm);
- setLops(rmm);
- }
-
- /**
- *
- * @param mmtsj
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsTSMM(MMTSJType mmtsj)
- throws HopsException, LopsException
- {
- Hop input = getInput().get(mmtsj.isLeft()?1:0);
-
- MMTSJ tsmm = new MMTSJ(input.constructLops(), getDataType(), getValueType(), ExecType.MR, mmtsj);
- tsmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(tsmm);
-
- Aggregate agg1 = new Aggregate(tsmm, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum but the inputs do not have correction values
- setLineNumbers(agg1);
-
- setLops(agg1);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructMRLopsPMM()
- throws HopsException, LopsException
- {
- //PMM has two potential modes (a) w/ full permutation matrix input, and
- //(b) w/ already condensed input vector of target row positions.
-
- Hop pmInput = getInput().get(0);
- Hop rightInput = getInput().get(1);
- long brlen = pmInput.getRowsInBlock();
- long bclen = pmInput.getColsInBlock();
-
- //a) full permutation matrix input (potentially without empty block materialized)
- Lop lpmInput = pmInput.constructLops();
- if( pmInput.getDim2() != 1 ) //not a vector
- {
- //compute condensed permutation matrix vector input
- //v = rowMaxIndex(t(pm)) * rowMax(t(pm))
- ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
- transpose.setForcedExecType(ExecType.MR);
- HopRewriteUtils.copyLineNumbers(this, transpose);
-
- AggUnaryOp agg1 = new AggUnaryOp("tmp2a", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAXINDEX, Direction.Row, transpose);
- HopRewriteUtils.setOutputBlocksizes(agg1, brlen, bclen);
- agg1.refreshSizeInformation();
- agg1.setForcedExecType(ExecType.MR);
- HopRewriteUtils.copyLineNumbers(this, agg1);
-
- AggUnaryOp agg2 = new AggUnaryOp("tmp2b", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, Direction.Row, transpose);
- HopRewriteUtils.setOutputBlocksizes(agg2, brlen, bclen);
- agg2.refreshSizeInformation();
- agg2.setForcedExecType(ExecType.MR);
- HopRewriteUtils.copyLineNumbers(this, agg2);
-
- BinaryOp mult = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, agg1, agg2);
- HopRewriteUtils.setOutputBlocksizes(mult, brlen, bclen);
- mult.refreshSizeInformation();
- mult.setForcedExecType(ExecType.MR);
- //mult.computeMemEstimate(memo); //select exec type
- HopRewriteUtils.copyLineNumbers(this, mult);
-
- lpmInput = mult.constructLops();
-
- HopRewriteUtils.removeChildReference(pmInput, transpose);
- }
-
- //b) condensed permutation matrix vector input (target rows)
- Hop nrow = HopRewriteUtils.createValueHop(pmInput, true); //NROW
- HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0);
- nrow.setForcedExecType(ExecType.CP);
- HopRewriteUtils.copyLineNumbers(this, nrow);
- Lop lnrow = nrow.constructLops();
-
- boolean needPart = !pmInput.dimsKnown() || pmInput.getDim1() > DistributedCacheInput.PARTITION_SIZE;
- double mestPM = OptimizerUtils.estimateSize(pmInput.getDim1(), 1);
- if( needPart ){ //requires partitioning
- lpmInput = new DataPartition(lpmInput, DataType.MATRIX, ValueType.DOUBLE, (mestPM>OptimizerUtils.getLocalMemBudget())?ExecType.MR:ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- lpmInput.getOutputParameters().setDimensions(pmInput.getDim1(), 1, getRowsInBlock(), getColsInBlock(), pmInput.getDim1());
- setLineNumbers(lpmInput);
- }
-
- _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
- PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), lnrow, getDataType(), getValueType(), needPart, _outputEmptyBlocks, ExecType.MR);
- pmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pmm);
-
- Aggregate aggregate = new Aggregate(pmm, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
- aggregate.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- aggregate.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum but the inputs do not have correction values
- setLineNumbers(aggregate);
-
- setLops(aggregate);
-
- HopRewriteUtils.removeChildReference(pmInput, nrow);
- }
-
-
-
- /**
- * Determines if the rewrite t(X)%*%Y -> t(t(Y)%*%X) is applicable
- * and cost effective. Whenever X is a wide matrix and Y is a vector
- * this has huge impact, because the transpose of X would dominate
- * the entire operation costs.
- *
- * @return
- */
- private boolean isLeftTransposeRewriteApplicable(boolean CP, boolean checkMemMR)
- {
- //check for forced MR or Spark execution modes, which prevent the introduction of
- //additional CP operations and hence the rewrite application
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP //not hybrid_mr
- || DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK ) //not hybrid_spark
- {
- return false;
- }
-
- boolean ret = false;
- Hop h1 = getInput().get(0);
- Hop h2 = getInput().get(1);
-
- //check for known dimensions and cost for t(X) vs t(v) + t(tvX)
- //(for both CP/MR, we explicitly check that new transposes fit in memory,
- //even a ba in CP does not imply that both transposes can be executed in CP)
- if( CP ) //in-memory ba
- {
- if( h1 instanceof ReorgOp && ((ReorgOp)h1).getOp()==ReOrgOp.TRANSPOSE )
- {
- long m = h1.getDim1();
- long cd = h1.getDim2();
- long n = h2.getDim2();
-
- //check for known dimensions (necessary condition for subsequent checks)
- ret = (m>0 && cd>0 && n>0);
-
- //check operation memory with changed transpose (this is important if we have
- //e.g., t(X) %*% v, where X is sparse and tX fits in memory but X does not
- double memX = h1.getInput().get(0).getOutputMemEstimate();
- double memtv = OptimizerUtils.estimateSizeExactSparsity(n, cd, 1.0);
- double memtXv = OptimizerUtils.estimateSizeExactSparsity(n, m, 1.0);
- double newMemEstimate = memtv + memX + memtXv;
- ret &= ( newMemEstimate < OptimizerUtils.getLocalMemBudget() );
-
- //check for cost benefit of t(X) vs t(v) + t(tvX) and memory of additional transpose ops
- ret &= ( m*cd > (cd*n + m*n) &&
- 2 * OptimizerUtils.estimateSizeExactSparsity(cd, n, 1.0) < OptimizerUtils.getLocalMemBudget() &&
- 2 * OptimizerUtils.estimateSizeExactSparsity(m, n, 1.0) < OptimizerUtils.getLocalMemBudget() );
-
- //update operation memory estimate (e.g., for parfor optimizer)
- if( ret )
- _memEstimate = newMemEstimate;
- }
- }
- else //MR
- {
- if( h1 instanceof ReorgOp && ((ReorgOp)h1).getOp()==ReOrgOp.TRANSPOSE )
- {
- long m = h1.getDim1();
- long cd = h1.getDim2();
- long n = h2.getDim2();
-
-
- //note: output size constraint for mapmult already checked by optfindmmultmethod
- if( m>0 && cd>0 && n>0 && (m*cd > (cd*n + m*n)) &&
- 2 * OptimizerUtils.estimateSizeExactSparsity(cd, n, 1.0) < OptimizerUtils.getLocalMemBudget() &&
- 2 * OptimizerUtils.estimateSizeExactSparsity(m, n, 1.0) < OptimizerUtils.getLocalMemBudget() &&
- (!checkMemMR || OptimizerUtils.estimateSizeExactSparsity(cd, n, 1.0) < OptimizerUtils.getRemoteMemBudgetMap(true)) )
- {
- ret = true;
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param X
- * @param Y
- * @return
- */
- private MMCJType getMMCJAggregationType(Hop X, Hop Y)
- {
- //choose quickpath (no aggregation) if the aggregation buffer likely has to spill and the smaller block fits
- //into the minimal cache size and hence is guaranteed not to require spilling
- double sizeX = OptimizerUtils.estimateSize(X.getDim1(), Math.min(X.getDim2(), X.getColsInBlock()));
- double sizeY = OptimizerUtils.estimateSize(Math.min(Y.getDim1(), Y.getRowsInBlock()), Y.getDim2());
-
- return (dimsKnown() && 2*OptimizerUtils.estimateSize(getDim1(), getDim2())>OptimizerUtils.getRemoteMemBudgetReduce()
- && ( sizeX < MMCJMRReducerWithAggregator.MIN_CACHE_SIZE
- || sizeY < MMCJMRReducerWithAggregator.MIN_CACHE_SIZE) )
- ? MMCJType.NO_AGG : MMCJType.AGG ;
- }
-
- /**
- *
- * @param agg
- * @return
- */
- private SparkAggType getSparkMMAggregationType( boolean agg )
- {
- if( !agg )
- return SparkAggType.NONE;
-
- if( dimsKnown() && getDim1()<=getRowsInBlock() && getDim2()<=getColsInBlock() )
- return SparkAggType.SINGLE_BLOCK;
- else
- return SparkAggType.MULTI_BLOCK;
- }
-
- /**
- *
- * @param method
- * @return
- */
- private boolean requiresAggregation(MMultMethod method)
- {
- //worst-case assumption (for plan correctness)
- boolean ret = true;
-
- //right side cached (no agg if left has just one column block)
- if( method == MMultMethod.MAPMM_R && getInput().get(0).getDim2() > 0 //known num columns
- && getInput().get(0).getDim2() <= getInput().get(0).getColsInBlock() )
- {
- ret = false;
- }
-
- //left side cached (no agg if right has just one row block)
- if( method == MMultMethod.MAPMM_L && getInput().get(1).getDim1() > 0 //known num rows
- && getInput().get(1).getDim1() <= getInput().get(1).getRowsInBlock() )
- {
- ret = false;
- }
-
- return ret;
- }
-
- /**
- *
- * @param method
- * @return
- */
- private boolean requiresPartitioning(MMultMethod method, boolean rewrite)
- {
- boolean ret = true; //worst-case
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
-
- //right side cached
- if( method == MMultMethod.MAPMM_R && input2.dimsKnown() ) { //known input size
- ret = (input2.getDim1()*input2.getDim2() > DistributedCacheInput.PARTITION_SIZE);
-
- //conservative: do not apply partitioning if this forces t(X) into separate job
- //if( !rewrite && input1 instanceof ReorgOp && ((ReorgOp)input1).getOp()==ReOrgOp.TRANSPOSE )
- // ret = false;
- }
-
- //left side cached (no agg if right has just one row block)
- if( method == MMultMethod.MAPMM_L && input1.dimsKnown() ) { //known input size
- ret = (input1.getDim1()*input1.getDim2() > DistributedCacheInput.PARTITION_SIZE);
-
- //conservative: do not apply partitioning if this forces t(X) into separate job
- //if( !rewrite && input2 instanceof ReorgOp && ((ReorgOp)input2).getOp()==ReOrgOp.TRANSPOSE )
- // ret = false;
- }
-
- return ret;
- }
-
- /**
- * Estimates the memory footprint of MapMult operation depending on which input is put into distributed cache.
- * This function is called by <code>optFindMMultMethod()</code> to decide the execution strategy, as well as by
- * piggybacking to decide the number of Map-side instructions to put into a single GMR job.
- */
- public static double getMapmmMemEstimate(long m1_rows, long m1_cols, long m1_rpb, long m1_cpb, long m1_nnz,
- long m2_rows, long m2_cols, long m2_rpb, long m2_cpb, long m2_nnz, int cachedInputIndex, boolean pmm)
- {
- // If the size of one input is small, choose a method that uses distributed cache
- // NOTE: be aware of output size because one input block might generate many output blocks
- double m1SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz); //m1 partitioned
- double m2SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz); //m2 partitioned
-
- double m1BlockSize = OptimizerUtils.estimateSize(Math.min(m1_rows, m1_rpb), Math.min(m1_cols, m1_cpb));
- double m2BlockSize = OptimizerUtils.estimateSize(Math.min(m2_rows, m2_rpb), Math.min(m2_cols, m2_cpb));
- double m3m1OutSize = OptimizerUtils.estimateSize(Math.min(m1_rows, m1_rpb), m2_cols); //output per m1 block if m2 in cache
- double m3m2OutSize = OptimizerUtils.estimateSize(m1_rows, Math.min(m2_cols, m2_cpb)); //output per m2 block if m1 in cache
-
- double footprint = 0;
- if( pmm )
- {
- //permutation matrix multiply
- //(one input block -> at most two output blocks)
- footprint = m1SizeP + 3*m2BlockSize; //in+2*out
- }
- else
- {
- //generic matrix multiply
- if ( cachedInputIndex == 1 ) {
- // left input (m1) is in cache
- footprint = m1SizeP+m2BlockSize+m3m2OutSize;
- }
- else {
- // right input (m2) is in cache
- footprint = m1BlockSize+m2SizeP+m3m1OutSize;
- }
- }
-
- return footprint;
- }
-
- /**
- * Optimization that chooses between two methods to perform matrix multiplication on map-reduce.
- *
- * More details on the cost-model used: refer ICDE 2011 paper.
- */
- private static MMultMethod optFindMMultMethodMR ( long m1_rows, long m1_cols, long m1_rpb, long m1_cpb, long m1_nnz,
- long m2_rows, long m2_cols, long m2_rpb, long m2_cpb, long m2_nnz,
- MMTSJType mmtsj, ChainType chainType, boolean leftPMInput )
- {
- double memBudget = MAPMULT_MEM_MULTIPLIER * OptimizerUtils.getRemoteMemBudgetMap(true);
-
- // Step 0: check for forced mmultmethod
- if( FORCED_MMULT_METHOD !=null )
- return FORCED_MMULT_METHOD;
-
- // Step 1: check TSMM
- // If transpose self pattern and result is single block:
- // use specialized TSMM method (always better than generic jobs)
- if( ( mmtsj == MMTSJType.LEFT && m2_cols>=0 && m2_cols <= m2_cpb )
- || ( mmtsj == MMTSJType.RIGHT && m1_rows>=0 && m1_rows <= m1_rpb ) )
- {
- return MMultMethod.TSMM;
- }
-
- // Step 2: check MapMultChain
- // If mapmultchain pattern and result is a single block:
- // use specialized mapmult method
- if( OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES )
- {
- //matmultchain if dim2(X)<=blocksize and all vectors fit in mappers
- //(X: m1_cols x m1_rows, v: m1_rows x m2_cols, w: m1_cols x m2_cols)
- //NOTE: generalization possibe: m2_cols>=0 && m2_cols<=m2_cpb
- if( chainType!=ChainType.NONE && m1_rows>=0 && m1_rows<= m1_rpb && m2_cols==1 )
- {
- if( chainType==ChainType.XtXv && m1_rows>=0 && m2_cols>=0
- && OptimizerUtils.estimateSize(m1_rows, m2_cols ) < memBudget )
- {
- return MMultMethod.MAPMM_CHAIN;
- }
- else if( chainType==ChainType.XtwXv && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
- && OptimizerUtils.estimateSize(m1_rows, m2_cols )
- + OptimizerUtils.estimateSize(m1_cols, m2_cols) < memBudget )
- {
- return MMultMethod.MAPMM_CHAIN;
- }
- }
- }
-
- // Step 3: check for PMM (permutation matrix needs to fit into mapper memory)
- // (needs to be checked before mapmult for consistency with removeEmpty compilation
- double footprintPM1 = getMapmmMemEstimate(m1_rows, 1, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, true);
- double footprintPM2 = getMapmmMemEstimate(m2_rows, 1, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, true);
- if( (footprintPM1 < memBudget && m1_rows>=0 || footprintPM2 < memBudget && m2_rows>=0 )
- && leftPMInput )
- {
- return MMultMethod.PMM;
- }
-
- // Step 4: check MapMult
- // If the size of one input is small, choose a method that uses distributed cache
- // (with awareness of output size because one input block might generate many output blocks)
- //memory estimates for local partitioning (mb -> partitioned mb)
- double m1SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz); //m1 partitioned
- double m2SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz); //m2 partitioned
-
- //memory estimates for remote execution (broadcast and outputs)
- double footprint1 = getMapmmMemEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, false);
- double footprint2 = getMapmmMemEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 2, false);
-
- if ( (footprint1 < memBudget && m1_rows>=0 && m1_cols>=0)
- || (footprint2 < memBudget && m2_rows>=0 && m2_cols>=0) )
- {
- //apply map mult if one side fits in remote task memory
- //(if so pick smaller input for distributed cache)
- if( m1SizeP < m2SizeP && m1_rows>=0 && m1_cols>=0)
- return MMultMethod.MAPMM_L;
- else
- return MMultMethod.MAPMM_R;
- }
-
- // Step 5: check for unknowns
- // If the dimensions are unknown at compilation time, simply assume
- // the worst-case scenario and produce the most robust plan -- which is CPMM
- if ( m1_rows == -1 || m1_cols == -1 || m2_rows == -1 || m2_cols == -1 )
- return MMultMethod.CPMM;
-
- // Step 6: Decide CPMM vs RMM based on io costs
-
- //estimate shuffle costs weighted by parallelism
- double rmm_costs = getRMMCostEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m2_rows, m2_cols, m2_rpb, m2_cpb);
- double cpmm_costs = getCPMMCostEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m2_rows, m2_cols, m2_rpb, m2_cpb);
-
- //final mmult method decision
- if ( cpmm_costs < rmm_costs )
- return MMultMethod.CPMM;
- else
- return MMultMethod.RMM;
- }
-
- /**
- *
- * @param m1_rows
- * @param m1_cols
- * @param m2_rows
- * @param m2_cols
- * @param mmtsj
- * @param chainType
- * @return
- */
- private static MMultMethod optFindMMultMethodCP( long m1_rows, long m1_cols, long m2_rows, long m2_cols, MMTSJType mmtsj, ChainType chainType, boolean leftPM )
- {
- //step 1: check for TSMM pattern
- if( mmtsj != MMTSJType.NONE )
- return MMultMethod.TSMM;
-
- //step 2: check for MMChain pattern
- if( chainType != ChainType.NONE && OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES && m2_cols==1 )
- return MMultMethod.MAPMM_CHAIN;
-
- //step 3: check for PMM
- if( leftPM && m1_cols==1 && m2_rows!=1 )
- return MMultMethod.PMM;
-
- //step 4: general purpose MM
- return MMultMethod.MM;
- }
-
- /**
- *
- * @param m1_rows
- * @param m1_cols
- * @param m1_rpb
- * @param m1_cpb
- * @param m2_rows
- * @param m2_cols
- * @param m2_rpb
- * @param m2_cpb
- * @param mmtsj
- * @param chainType
- * @return
- */
- private MMultMethod optFindMMultMethodSpark( long m1_rows, long m1_cols, long m1_rpb, long m1_cpb, long m1_nnz,
- long m2_rows, long m2_cols, long m2_rpb, long m2_cpb, long m2_nnz,
- MMTSJType mmtsj, ChainType chainType, boolean leftPMInput, boolean tmmRewrite )
- {
- //Notes: Any broadcast needs to fit twice in local memory because we partition the input in cp,
- //and needs to fit once in executor broadcast memory. The 2GB broadcast constraint is no longer
- //required because the max_int byte buffer constraint has been fixed in Spark 1.4
- double memBudgetExec = MAPMULT_MEM_MULTIPLIER * SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- //reset spark broadcast memory information (for concurrent parfor jobs, awareness of additional
- //cp memory requirements on spark rdd operations with broadcasts)
- _spBroadcastMemEstimate = 0;
-
- // Step 0: check for forced mmultmethod
- if( FORCED_MMULT_METHOD !=null )
- return FORCED_MMULT_METHOD;
-
- // Step 1: check TSMM
- // If transpose self pattern and result is single block:
- // use specialized TSMM method (always better than generic jobs)
- if( ( mmtsj == MMTSJType.LEFT && m2_cols>=0 && m2_cols <= m2_cpb )
- || ( mmtsj == MMTSJType.RIGHT && m1_rows>=0 && m1_rows <= m1_rpb ) )
- {
- return MMultMethod.TSMM;
- }
-
- // Step 2: check MapMMChain
- // If mapmultchain pattern and result is a single block:
- // use specialized mapmult method
- if( OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES )
- {
- //matmultchain if dim2(X)<=blocksize and all vectors fit in mappers
- //(X: m1_cols x m1_rows, v: m1_rows x m2_cols, w: m1_cols x m2_cols)
- //NOTE: generalization possibe: m2_cols>=0 && m2_cols<=m2_cpb
- if( chainType!=ChainType.NONE && m1_rows >=0 && m1_rows <= m1_rpb && m2_cols==1 )
- {
- if( chainType==ChainType.XtXv && m1_rows>=0 && m2_cols>=0
- && OptimizerUtils.estimateSize(m1_rows, m2_cols ) < memBudgetExec )
- {
- return MMultMethod.MAPMM_CHAIN;
- }
- else if( chainType==ChainType.XtwXv && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
- && OptimizerUtils.estimateSize(m1_rows, m2_cols)
- + OptimizerUtils.estimateSize(m1_cols, m2_cols) < memBudgetExec
- && 2*(OptimizerUtils.estimateSize(m1_rows, m2_cols)
- + OptimizerUtils.estimateSize(m1_cols, m2_cols)) < memBudgetLocal )
- {
- _spBroadcastMemEstimate = 2*(OptimizerUtils.estimateSize(m1_rows, m2_cols)
- + OptimizerUtils.estimateSize(m1_cols, m2_cols));
- return MMultMethod.MAPMM_CHAIN;
- }
- }
- }
-
- // Step 3: check for PMM (permutation matrix needs to fit into mapper memory)
- // (needs to be checked before mapmult for consistency with removeEmpty compilation
- double footprintPM1 = getMapmmMemEstimate(m1_rows, 1, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, true);
- double footprintPM2 = getMapmmMemEstimate(m2_rows, 1, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, true);
- if( (footprintPM1 < memBudgetExec && m1_rows>=0 || footprintPM2 < memBudgetExec && m2_rows>=0)
- && 2*OptimizerUtils.estimateSize(m1_rows, 1) < memBudgetLocal
- && leftPMInput )
- {
- _spBroadcastMemEstimate = 2*OptimizerUtils.estimateSize(m1_rows, 1);
- return MMultMethod.PMM;
- }
-
- // Step 4: check MapMM
- // If the size of one input is small, choose a method that uses broadcast variables to prevent shuffle
-
- //memory estimates for local partitioning (mb -> partitioned mb)
- double m1Size = OptimizerUtils.estimateSizeExactSparsity(m1_rows, m1_cols, m1_nnz); //m1 single block
- double m2Size = OptimizerUtils.estimateSizeExactSparsity(m2_rows, m2_cols, m2_nnz); //m2 single block
- double m1SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz); //m1 partitioned
- double m2SizeP = OptimizerUtils.estimatePartitionedSizeExactSparsity(m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz); //m2 partitioned
-
- //memory estimates for remote execution (broadcast and outputs)
- double footprint1 = getMapmmMemEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 1, false);
- double footprint2 = getMapmmMemEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m1_nnz, m2_rows, m2_cols, m2_rpb, m2_cpb, m2_nnz, 2, false);
-
- if ( (footprint1 < memBudgetExec && m1Size+m1SizeP < memBudgetLocal && m1_rows>=0 && m1_cols>=0)
- || (footprint2 < memBudgetExec && m2Size+m2SizeP < memBudgetLocal && m2_rows>=0 && m2_cols>=0) )
- {
- //apply map mult if one side fits in remote task memory
- //(if so pick smaller input for distributed cache)
- if( m1SizeP < m2SizeP && m1_rows>=0 && m1_cols>=0) {
- _spBroadcastMemEstimate = m1Size+m1SizeP;
- return MMultMethod.MAPMM_L;
- }
- else {
- _spBroadcastMemEstimate = m2Size+m2SizeP;
- return MMultMethod.MAPMM_R;
- }
- }
-
- // Step 5: check for unknowns
- // If the dimensions are unknown at compilation time, simply assume
- // the worst-case scenario and produce the most robust plan -- which is CPMM
- if ( m1_rows == -1 || m1_cols == -1 || m2_rows == -1 || m2_cols == -1 )
- return MMultMethod.CPMM;
-
- // Step 6: check for ZIPMM
- // If t(X)%*%y -> t(t(y)%*%X) rewrite and ncol(X)<blocksize
- if( tmmRewrite && m1_rows >= 0 && m1_rows <= m1_rpb //blocksize constraint left
- && m2_cols >= 0 && m2_cols <= m2_cpb ) //blocksize constraint right
- {
- return MMultMethod.ZIPMM;
- }
-
- // Step 7: Decide CPMM vs RMM based on io costs
- //estimate shuffle costs weighted by parallelism
- //TODO currently we reuse the mr estimates, these need to be fine-tune for our spark operators
- double rmm_costs = getRMMCostEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m2_rows, m2_cols, m2_rpb, m2_cpb);
- double cpmm_costs = getCPMMCostEstimate(m1_rows, m1_cols, m1_rpb, m1_cpb, m2_rows, m2_cols, m2_rpb, m2_cpb);
-
- //final mmult method decision
- if ( cpmm_costs < rmm_costs )
- return MMultMethod.CPMM;
- else
- return MMultMethod.RMM;
- }
-
- /**
- *
- * @param m1_rows
- * @param m1_cols
- * @param m1_rpb
- * @param m1_cpb
- * @param m2_rows
- * @param m2_cols
- * @param m2_rpb
- * @param m2_cpb
- * @return
- */
- private static double getRMMCostEstimate( long m1_rows, long m1_cols, long m1_rpb, long m1_cpb,
- long m2_rows, long m2_cols, long m2_rpb, long m2_cpb )
- {
- long m1_nrb = (long) Math.ceil((double)m1_rows/m1_rpb); // number of row blocks in m1
- long m2_ncb = (long) Math.ceil((double)m2_cols/m2_cpb); // number of column blocks in m2
-
- // TODO: we must factor in the "sparsity"
- double m1_size = m1_rows * m1_cols;
- double m2_size = m2_rows * m2_cols;
- double result_size = m1_rows * m2_cols;
-
- int numReducersRMM = OptimizerUtils.getNumReducers(true);
-
- // Estimate the cost of RMM
- // RMM phase 1
- double rmm_shuffle = (m2_ncb*m1_size) + (m1_nrb*m2_size);
- double rmm_io = m1_size + m2_size + result_size;
- double rmm_nred = Math.min( m1_nrb * m2_ncb, //max used reducers
- numReducersRMM); //available reducers
- // RMM total costs
- double rmm_costs = (rmm_shuffle + rmm_io) / rmm_nred;
-
- // return total costs
- return rmm_costs;
- }
-
- /**
- *
- * @param m1_rows
- * @param m1_cols
- * @param m1_rpb
- * @param m1_cpb
- * @param m2_rows
- * @param m2_cols
- * @param m2_rpb
- * @param m2_cpb
- * @return
- */
- private static double getCPMMCostEstimate( long m1_rows, long m1_cols, long m1_rpb, long m1_cpb,
- long m2_rows, long m2_cols, long m2_rpb, long m2_cpb )
- {
- long m1_nrb = (long) Math.ceil((double)m1_rows/m1_rpb); // number of row blocks in m1
- long m1_ncb = (long) Math.ceil((double)m1_cols/m1_cpb); // number of column blocks in m1
- long m2_ncb = (long) Math.ceil((double)m2_cols/m2_cpb); // number of column blocks in m2
-
- // TODO: we must factor in the "sparsity"
- double m1_size = m1_rows * m1_cols;
- double m2_size = m2_rows * m2_cols;
- double result_size = m1_rows * m2_cols;
-
- int numReducersCPMM = OptimizerUtils.getNumReducers(false);
-
- // Estimate the cost of CPMM
- // CPMM phase 1
- double cpmm_shuffle1 = m1_size + m2_size;
- double cpmm_nred1 = Math.min( m1_ncb, //max used reducers
- numReducersCPMM); //available reducers
- double cpmm_io1 = m1_size + m2_size + cpmm_nred1 * result_size;
- // CPMM phase 2
- double cpmm_shuffle2 = cpmm_nred1 * result_size;
- double cpmm_io2 = cpmm_nred1 * result_size + result_size;
- double cpmm_nred2 = Math.min( m1_nrb * m2_ncb, //max used reducers
- numReducersCPMM); //available reducers
- // CPMM total costs
- double cpmm_costs = (cpmm_shuffle1+cpmm_io1)/cpmm_nred1 //cpmm phase1
- +(cpmm_shuffle2+cpmm_io2)/cpmm_nred2; //cpmm phase2
-
- //return total costs
- return cpmm_costs;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
-
- if( isMatrixMultiply() )
- {
- setDim1(input1.getDim1());
- setDim2(input2.getDim2());
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- AggBinaryOp ret = new AggBinaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret.innerOp = innerOp;
- ret.outerOp = outerOp;
- ret._hasLeftPMInput = _hasLeftPMInput;
- ret._maxNumThreads = _maxNumThreads;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof AggBinaryOp) )
- return false;
-
- AggBinaryOp that2 = (AggBinaryOp)that;
- return ( innerOp == that2.innerOp
- && outerOp == that2.outerOp
- && getInput().get(0) == that2.getInput().get(0)
- && getInput().get(1) == that2.getInput().get(1)
- && _hasLeftPMInput == that2._hasLeftPMInput
- && _maxNumThreads == that2._maxNumThreads);
- }
-}
[64/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
index 4278806..19e497e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
@@ -26,11 +26,11 @@ import org.apache.spark.mllib.linalg.SparseMatrix;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>, Tuple2<Object, Object>, Matrix> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/IsBlockInRange.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/IsBlockInRange.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/IsBlockInRange.java
index 208eab4..ff3ec30 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/IsBlockInRange.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/IsBlockInRange.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class IsBlockInRange implements Function<Tuple2<MatrixIndexes,MatrixBlock>, Boolean>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
index 496f433..5d5b99e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Function class that finds the index of last cell in the given matrix block.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixMatrixBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixMatrixBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixMatrixBinaryOpFunction.java
index fed5095..08a201b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixMatrixBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixMatrixBinaryOpFunction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class MatrixMatrixBinaryOpFunction implements Function<Tuple2<MatrixBlock,MatrixBlock>, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixScalarUnaryFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixScalarUnaryFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixScalarUnaryFunction.java
index 509bda3..3b1e9ec 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixScalarUnaryFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixScalarUnaryFunction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
index 8a551fe..6eeed63 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class MatrixVectorBinaryOpFunction implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes,MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
index e810a84..2e0c6ba 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.Iterator;
@@ -23,12 +23,12 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunction<Iterator<Tuple2<MatrixIndexes,MatrixBlock>>, MatrixIndexes,MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
index 70b863c..8f1c225 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.Iterator;
@@ -23,10 +23,10 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class OuterVectorBinaryOpFunction implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes,MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInCombiner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInCombiner.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInCombiner.java
index e9a8cbe..f5b92f1 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInCombiner.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInCombiner.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.Function2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class PerformGroupByAggInCombiner implements Function2<WeightedCell, WeightedCell, WeightedCell> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInReducer.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInReducer.java
index cd7656b..ff438b1 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/PerformGroupByAggInReducer.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class PerformGroupByAggInReducer implements PairFunction<Tuple2<Long,Iterable<WeightedCell>>, Long, WeightedCell> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java
index 03d6599..165ab34 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.IndexFunction;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.IndexFunction;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class ReorgMapFunction implements PairFunction<Tuple2<MatrixIndexes, MatrixBlock>, MatrixIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java
index 90903ca..0d56eea 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
@@ -23,8 +23,8 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
public class ReplicateVectorFunction implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes,MatrixBlock>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/SparkListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/SparkListener.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/SparkListener.java
index f7f8e27..9187a8f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/SparkListener.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/SparkListener.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
import java.util.HashMap;
@@ -37,9 +37,9 @@ import scala.collection.Iterator;
import scala.collection.Seq;
import scala.xml.Node;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
// Instead of extending org.apache.spark.JavaSparkListener
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/UnflattenIterablesAfterCogroup.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/UnflattenIterablesAfterCogroup.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/UnflattenIterablesAfterCogroup.java
index 80d18ba..5564786 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/UnflattenIterablesAfterCogroup.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/UnflattenIterablesAfterCogroup.java
@@ -1,11 +1,11 @@
-package com.ibm.bi.dml.runtime.instructions.spark.functions;
+package org.apache.sysml.runtime.instructions.spark.functions;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
public class UnflattenIterablesAfterCogroup implements PairFunction<Tuple2<MatrixIndexes,Tuple2<Iterable<MatrixBlock>,Iterable<MatrixBlock>>>, MatrixIndexes, Tuple2<MatrixBlock, MatrixBlock>> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
index 9caed7c..f73f36c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.utils;
+package org.apache.sysml.runtime.instructions.spark.utils;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.CorrMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RowMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.instructions.spark.data.CorrMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.data.RowMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
/**
* Collection of utility methods for aggregating binary block rdds. As a general
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
index ae67a27..74fb234 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.utils;
+package org.apache.sysml.runtime.instructions.spark.utils;
import java.io.IOException;
import java.io.Serializable;
@@ -37,21 +37,21 @@ import org.apache.spark.mllib.regression.LabeledPoint;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.data.SerLongWritable;
-import com.ibm.bi.dml.runtime.instructions.spark.data.SerText;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.ReblockBuffer;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.data.SerLongWritable;
+import org.apache.sysml.runtime.instructions.spark.data.SerText;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.ReblockBuffer;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class RDDConverterUtils
{
@@ -209,8 +209,8 @@ public class RDDConverterUtils
/**
* Example usage:
* <pre><code>
- * import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils
- * import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics
+ * import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils
+ * import org.apache.sysml.runtime.matrix.MatrixCharacteristics
* import org.apache.spark.api.java.JavaSparkContext
* val A = sc.textFile("ranA.csv")
* val Amc = new MatrixCharacteristics
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
index 10dcce8..a85c022 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.utils;
+package org.apache.sysml.runtime.instructions.spark.utils;
import java.io.IOException;
import java.io.Serializable;
@@ -45,19 +45,19 @@ import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
-import com.ibm.bi.dml.api.MLOutput.ConvertDoubleArrayToRows;
-import com.ibm.bi.dml.api.MLOutput.ProjectRows;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ConvertMatrixBlockToIJVLines;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.ReblockBuffer;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.api.MLOutput.ConvertDoubleArrayToRows;
+import org.apache.sysml.api.MLOutput.ProjectRows;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.functions.ConvertMatrixBlockToIJVLines;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.ReblockBuffer;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* NOTE: These are experimental converter utils. Once thoroughly tested, they
@@ -77,8 +77,8 @@ public class RDDConverterUtilsExt
/**
* Example usage:
* <pre><code>
- * import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt
- * import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics
+ * import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt
+ * import org.apache.sysml.runtime.matrix.MatrixCharacteristics
* import org.apache.spark.api.java.JavaSparkContext
* import org.apache.spark.mllib.linalg.distributed.MatrixEntry
* import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
index 78cb0eb..d5c82f0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark.utils;
+package org.apache.sysml.runtime.instructions.spark.utils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -32,19 +32,19 @@ import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.functionobjects.SortIndex;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RowMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ReplicateVectorFunction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.functionobjects.SortIndex;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.data.RowMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.functions.ReplicateVectorFunction;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index 1fcf9c2..c504e47 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.instructions.spark.utils;
+package org.apache.sysml.runtime.instructions.spark.utils;
import java.util.ArrayList;
import java.util.Collections;
@@ -30,17 +30,17 @@ import org.apache.spark.storage.StorageLevel;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockFunction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.Checkpoint;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockFunction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class SparkUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/BinaryBlockSerialization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/BinaryBlockSerialization.java b/src/main/java/org/apache/sysml/runtime/io/BinaryBlockSerialization.java
index ef63807..d82fb32 100644
--- a/src/main/java/org/apache/sysml/runtime/io/BinaryBlockSerialization.java
+++ b/src/main/java/org/apache/sysml/runtime/io/BinaryBlockSerialization.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.io.InputStream;
@@ -25,13 +25,13 @@ import org.apache.hadoop.io.serializer.Deserializer;
import org.apache.hadoop.io.serializer.Serialization;
import org.apache.hadoop.io.serializer.Serializer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataInputStream;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
/**
* This custom serialization class can be used via
- * job.set("io.serializations", "com.ibm.bi.dml.runtime.io.BinaryBlockSerialization");
+ * job.set("io.serializations", "org.apache.sysml.runtime.io.BinaryBlockSerialization");
*
*
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 000ddb9..d9abcb3 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -29,7 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.RecordReader;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class IOUtilFunctions
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
index 529e3c3..38b8c38 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.EOFException;
import java.io.IOException;
@@ -25,10 +25,10 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* Base class for all format-specific matrix readers. Every reader is required to implement the basic
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
index cbeda73..fcbe71c 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/MatrixWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixWriter.java b/src/main/java/org/apache/sysml/runtime/io/MatrixWriter.java
index 87cf001..5179d70 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixWriter.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixWriter.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* Base class for all format-specific matrix writers. Every writer is required to implement the basic
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
index 0f835c4..8df773d 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReadProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReadProperties.java b/src/main/java/org/apache/sysml/runtime/io/ReadProperties.java
index 6faf9bb..9bf7b9f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReadProperties.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReadProperties.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
public class ReadProperties
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java
index 154c0d1..0ad0d18 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,12 +26,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class ReaderBinaryBlock extends MatrixReader
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
index 551203b..59e1912 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,12 +30,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class ReaderBinaryBlockParallel extends ReaderBinaryBlock
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java
index 46f8875..9637d52 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
@@ -24,11 +24,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java
index 67ee2ce..b659826 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedReader;
import java.io.IOException;
@@ -30,12 +30,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ReaderTextCSV extends MatrixReader
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
index 9b749c1..5c7de41 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
@@ -38,11 +38,11 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* Parallel version of ReaderTextCSV.java. To summarize, we do two passes in
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
index 4064148..cadd70b 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedReader;
import java.io.IOException;
@@ -33,11 +33,11 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
public class ReaderTextCell extends MatrixReader
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
index 4eb703a..2af5472 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
@@ -34,13 +34,13 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* Parallel version of ReaderTextCell.java. To summarize, we create read tasks per split
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
index 1712b51..6fdb287 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.File;
import java.io.IOException;
@@ -25,15 +25,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterBinaryBlock extends MatrixWriter
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
index 0218ffd..440bbf4 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,15 +30,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterBinaryBlockParallel extends WriterBinaryBlock
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
index 9ca2254..4129b6d 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.IOException;
@@ -24,15 +24,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterBinaryCell extends MatrixWriter
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
index 7b7c7ca..dfc4b8d 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -31,13 +31,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
index 89384b3..cbd1017 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -31,14 +31,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
index dec3c8d..3f83c42 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -33,14 +33,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
index d76894c..c20b482 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -31,14 +31,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.CSVFileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
index 6cb81d3..94ae87e 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -26,13 +26,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterTextCell extends MatrixWriter
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
index cacbe49..d77dd4e 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.io;
+package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -31,14 +31,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterTextCellParallel extends WriterTextCell
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
index 1a3538d..c480054 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.HashSet;
@@ -26,15 +26,15 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.CM_N_COVCell;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CMCOVMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.CMCOVMRReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.data.CM_N_COVCell;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CMCOVMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.CMCOVMRReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
public class CMCOVMR
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
index b8e8d23..b30c45f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.DataInput;
import java.io.DataOutput;
@@ -39,22 +39,22 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVAssignRowIDMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVAssignRowIDReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.runtime.transform.TfUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CSVAssignRowIDMapper;
+import org.apache.sysml.runtime.matrix.mapred.CSVAssignRowIDReducer;
+import org.apache.sysml.runtime.matrix.mapred.CSVReblockMapper;
+import org.apache.sysml.runtime.matrix.mapred.CSVReblockReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.runtime.transform.TfUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/CleanupMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CleanupMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CleanupMR.java
index 36c8df1..1dbc3a7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CleanupMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CleanupMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -39,12 +39,12 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.lib.NLineInputFormat;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.LocalFileUtils;
public class CleanupMR
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
index ca42a0f..1c50e42 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,25 +33,25 @@ import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.RunningJob;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineTernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedPair;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.ReduceBase;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineTernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixCell;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.matrix.data.WeightedPair;
+import org.apache.sysml.runtime.matrix.mapred.GMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.ReduceBase;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class CombineMR
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
index e910db6..3042f6c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.PrintWriter;
import java.util.HashSet;
@@ -33,33 +33,33 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRCombiner;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.DataGenMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.LibMatrixDatagen;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.mapred.GMRCombiner;
+import org.apache.sysml.runtime.matrix.mapred.GMRReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.DataGenMapper;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/DataPartitionMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataPartitionMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataPartitionMR.java
index c97a72b..968a07a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/DataPartitionMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/DataPartitionMR.java
@@ -16,17 +16,17 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerRemoteMR;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitioner;
+import org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteMR;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
public class DataPartitionMR
{
[76/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 3545ba3..19a5df5 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,27 +23,27 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.CSVReBlock;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.CSVReBlock;
+import org.apache.sysml.lops.Checkpoint;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.ReBlock;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.UtilFunctions;
public abstract class Hop
@@ -1100,180 +1100,180 @@ public abstract class Hop
RowCol, Row, Col
};
- protected static final HashMap<DataOpTypes, com.ibm.bi.dml.lops.Data.OperationTypes> HopsData2Lops;
+ protected static final HashMap<DataOpTypes, org.apache.sysml.lops.Data.OperationTypes> HopsData2Lops;
static {
- HopsData2Lops = new HashMap<Hop.DataOpTypes, com.ibm.bi.dml.lops.Data.OperationTypes>();
- HopsData2Lops.put(DataOpTypes.PERSISTENTREAD, com.ibm.bi.dml.lops.Data.OperationTypes.READ);
- HopsData2Lops.put(DataOpTypes.PERSISTENTWRITE, com.ibm.bi.dml.lops.Data.OperationTypes.WRITE);
- HopsData2Lops.put(DataOpTypes.TRANSIENTWRITE, com.ibm.bi.dml.lops.Data.OperationTypes.WRITE);
- HopsData2Lops.put(DataOpTypes.TRANSIENTREAD, com.ibm.bi.dml.lops.Data.OperationTypes.READ);
+ HopsData2Lops = new HashMap<Hop.DataOpTypes, org.apache.sysml.lops.Data.OperationTypes>();
+ HopsData2Lops.put(DataOpTypes.PERSISTENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ);
+ HopsData2Lops.put(DataOpTypes.PERSISTENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE);
+ HopsData2Lops.put(DataOpTypes.TRANSIENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE);
+ HopsData2Lops.put(DataOpTypes.TRANSIENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ);
}
- protected static final HashMap<Hop.AggOp, com.ibm.bi.dml.lops.Aggregate.OperationTypes> HopsAgg2Lops;
+ protected static final HashMap<Hop.AggOp, org.apache.sysml.lops.Aggregate.OperationTypes> HopsAgg2Lops;
static {
- HopsAgg2Lops = new HashMap<Hop.AggOp, com.ibm.bi.dml.lops.Aggregate.OperationTypes>();
- HopsAgg2Lops.put(AggOp.SUM, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanSum);
- HopsAgg2Lops.put(AggOp.SUM_SQ, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanSumSq);
- HopsAgg2Lops.put(AggOp.TRACE, com.ibm.bi.dml.lops.Aggregate.OperationTypes.KahanTrace);
- HopsAgg2Lops.put(AggOp.MIN, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Min);
- HopsAgg2Lops.put(AggOp.MAX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Max);
- HopsAgg2Lops.put(AggOp.MAXINDEX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.MaxIndex);
- HopsAgg2Lops.put(AggOp.MININDEX, com.ibm.bi.dml.lops.Aggregate.OperationTypes.MinIndex);
- HopsAgg2Lops.put(AggOp.PROD, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Product);
- HopsAgg2Lops.put(AggOp.MEAN, com.ibm.bi.dml.lops.Aggregate.OperationTypes.Mean);
- }
-
- protected static final HashMap<ReOrgOp, com.ibm.bi.dml.lops.Transform.OperationTypes> HopsTransf2Lops;
+ HopsAgg2Lops = new HashMap<Hop.AggOp, org.apache.sysml.lops.Aggregate.OperationTypes>();
+ HopsAgg2Lops.put(AggOp.SUM, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSum);
+ HopsAgg2Lops.put(AggOp.SUM_SQ, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSumSq);
+ HopsAgg2Lops.put(AggOp.TRACE, org.apache.sysml.lops.Aggregate.OperationTypes.KahanTrace);
+ HopsAgg2Lops.put(AggOp.MIN, org.apache.sysml.lops.Aggregate.OperationTypes.Min);
+ HopsAgg2Lops.put(AggOp.MAX, org.apache.sysml.lops.Aggregate.OperationTypes.Max);
+ HopsAgg2Lops.put(AggOp.MAXINDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MaxIndex);
+ HopsAgg2Lops.put(AggOp.MININDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MinIndex);
+ HopsAgg2Lops.put(AggOp.PROD, org.apache.sysml.lops.Aggregate.OperationTypes.Product);
+ HopsAgg2Lops.put(AggOp.MEAN, org.apache.sysml.lops.Aggregate.OperationTypes.Mean);
+ }
+
+ protected static final HashMap<ReOrgOp, org.apache.sysml.lops.Transform.OperationTypes> HopsTransf2Lops;
static {
- HopsTransf2Lops = new HashMap<ReOrgOp, com.ibm.bi.dml.lops.Transform.OperationTypes>();
- HopsTransf2Lops.put(ReOrgOp.TRANSPOSE, com.ibm.bi.dml.lops.Transform.OperationTypes.Transpose);
- HopsTransf2Lops.put(ReOrgOp.DIAG, com.ibm.bi.dml.lops.Transform.OperationTypes.Diag);
- HopsTransf2Lops.put(ReOrgOp.RESHAPE, com.ibm.bi.dml.lops.Transform.OperationTypes.Reshape);
- HopsTransf2Lops.put(ReOrgOp.SORT, com.ibm.bi.dml.lops.Transform.OperationTypes.Sort);
+ HopsTransf2Lops = new HashMap<ReOrgOp, org.apache.sysml.lops.Transform.OperationTypes>();
+ HopsTransf2Lops.put(ReOrgOp.TRANSPOSE, org.apache.sysml.lops.Transform.OperationTypes.Transpose);
+ HopsTransf2Lops.put(ReOrgOp.DIAG, org.apache.sysml.lops.Transform.OperationTypes.Diag);
+ HopsTransf2Lops.put(ReOrgOp.RESHAPE, org.apache.sysml.lops.Transform.OperationTypes.Reshape);
+ HopsTransf2Lops.put(ReOrgOp.SORT, org.apache.sysml.lops.Transform.OperationTypes.Sort);
}
- protected static final HashMap<Hop.Direction, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes> HopsDirection2Lops;
+ protected static final HashMap<Hop.Direction, org.apache.sysml.lops.PartialAggregate.DirectionTypes> HopsDirection2Lops;
static {
- HopsDirection2Lops = new HashMap<Hop.Direction, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes>();
- HopsDirection2Lops.put(Direction.RowCol, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.RowCol);
- HopsDirection2Lops.put(Direction.Col, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.Col);
- HopsDirection2Lops.put(Direction.Row, com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes.Row);
+ HopsDirection2Lops = new HashMap<Hop.Direction, org.apache.sysml.lops.PartialAggregate.DirectionTypes>();
+ HopsDirection2Lops.put(Direction.RowCol, org.apache.sysml.lops.PartialAggregate.DirectionTypes.RowCol);
+ HopsDirection2Lops.put(Direction.Col, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Col);
+ HopsDirection2Lops.put(Direction.Row, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Row);
}
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Binary.OperationTypes> HopsOpOp2LopsB;
+ protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Binary.OperationTypes> HopsOpOp2LopsB;
static {
- HopsOpOp2LopsB = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Binary.OperationTypes>();
- HopsOpOp2LopsB.put(OpOp2.PLUS, com.ibm.bi.dml.lops.Binary.OperationTypes.ADD);
- HopsOpOp2LopsB.put(OpOp2.MINUS, com.ibm.bi.dml.lops.Binary.OperationTypes.SUBTRACT);
- HopsOpOp2LopsB.put(OpOp2.MULT, com.ibm.bi.dml.lops.Binary.OperationTypes.MULTIPLY);
- HopsOpOp2LopsB.put(OpOp2.DIV, com.ibm.bi.dml.lops.Binary.OperationTypes.DIVIDE);
- HopsOpOp2LopsB.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.Binary.OperationTypes.MODULUS);
- HopsOpOp2LopsB.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.Binary.OperationTypes.INTDIV);
- HopsOpOp2LopsB.put(OpOp2.MINUS1_MULT, com.ibm.bi.dml.lops.Binary.OperationTypes.MINUS1_MULTIPLY);
- HopsOpOp2LopsB.put(OpOp2.LESS, com.ibm.bi.dml.lops.Binary.OperationTypes.LESS_THAN);
- HopsOpOp2LopsB.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.GREATER, com.ibm.bi.dml.lops.Binary.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsB.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.EQUALS);
- HopsOpOp2LopsB.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.Binary.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsB.put(OpOp2.MIN, com.ibm.bi.dml.lops.Binary.OperationTypes.MIN);
- HopsOpOp2LopsB.put(OpOp2.MAX, com.ibm.bi.dml.lops.Binary.OperationTypes.MAX);
- HopsOpOp2LopsB.put(OpOp2.AND, com.ibm.bi.dml.lops.Binary.OperationTypes.OR);
- HopsOpOp2LopsB.put(OpOp2.OR, com.ibm.bi.dml.lops.Binary.OperationTypes.AND);
- HopsOpOp2LopsB.put(OpOp2.SOLVE, com.ibm.bi.dml.lops.Binary.OperationTypes.SOLVE);
- HopsOpOp2LopsB.put(OpOp2.POW, com.ibm.bi.dml.lops.Binary.OperationTypes.POW);
- HopsOpOp2LopsB.put(OpOp2.LOG, com.ibm.bi.dml.lops.Binary.OperationTypes.NOTSUPPORTED);
- }
-
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes> HopsOpOp2LopsBS;
+ HopsOpOp2LopsB = new HashMap<Hop.OpOp2, org.apache.sysml.lops.Binary.OperationTypes>();
+ HopsOpOp2LopsB.put(OpOp2.PLUS, org.apache.sysml.lops.Binary.OperationTypes.ADD);
+ HopsOpOp2LopsB.put(OpOp2.MINUS, org.apache.sysml.lops.Binary.OperationTypes.SUBTRACT);
+ HopsOpOp2LopsB.put(OpOp2.MULT, org.apache.sysml.lops.Binary.OperationTypes.MULTIPLY);
+ HopsOpOp2LopsB.put(OpOp2.DIV, org.apache.sysml.lops.Binary.OperationTypes.DIVIDE);
+ HopsOpOp2LopsB.put(OpOp2.MODULUS, org.apache.sysml.lops.Binary.OperationTypes.MODULUS);
+ HopsOpOp2LopsB.put(OpOp2.INTDIV, org.apache.sysml.lops.Binary.OperationTypes.INTDIV);
+ HopsOpOp2LopsB.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Binary.OperationTypes.MINUS1_MULTIPLY);
+ HopsOpOp2LopsB.put(OpOp2.LESS, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN);
+ HopsOpOp2LopsB.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN_OR_EQUALS);
+ HopsOpOp2LopsB.put(OpOp2.GREATER, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN);
+ HopsOpOp2LopsB.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN_OR_EQUALS);
+ HopsOpOp2LopsB.put(OpOp2.EQUAL, org.apache.sysml.lops.Binary.OperationTypes.EQUALS);
+ HopsOpOp2LopsB.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Binary.OperationTypes.NOT_EQUALS);
+ HopsOpOp2LopsB.put(OpOp2.MIN, org.apache.sysml.lops.Binary.OperationTypes.MIN);
+ HopsOpOp2LopsB.put(OpOp2.MAX, org.apache.sysml.lops.Binary.OperationTypes.MAX);
+ HopsOpOp2LopsB.put(OpOp2.AND, org.apache.sysml.lops.Binary.OperationTypes.OR);
+ HopsOpOp2LopsB.put(OpOp2.OR, org.apache.sysml.lops.Binary.OperationTypes.AND);
+ HopsOpOp2LopsB.put(OpOp2.SOLVE, org.apache.sysml.lops.Binary.OperationTypes.SOLVE);
+ HopsOpOp2LopsB.put(OpOp2.POW, org.apache.sysml.lops.Binary.OperationTypes.POW);
+ HopsOpOp2LopsB.put(OpOp2.LOG, org.apache.sysml.lops.Binary.OperationTypes.NOTSUPPORTED);
+ }
+
+ protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.BinaryScalar.OperationTypes> HopsOpOp2LopsBS;
static {
- HopsOpOp2LopsBS = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes>();
- HopsOpOp2LopsBS.put(OpOp2.PLUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.ADD);
- HopsOpOp2LopsBS.put(OpOp2.MINUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.SUBTRACT);
- HopsOpOp2LopsBS.put(OpOp2.MULT, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MULTIPLY);
- HopsOpOp2LopsBS.put(OpOp2.DIV, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.DIVIDE);
- HopsOpOp2LopsBS.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MODULUS);
- HopsOpOp2LopsBS.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.INTDIV);
- HopsOpOp2LopsBS.put(OpOp2.LESS, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LESS_THAN);
- HopsOpOp2LopsBS.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.GREATER, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsBS.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsBS.put(OpOp2.MIN, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MIN);
- HopsOpOp2LopsBS.put(OpOp2.MAX, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.MAX);
- HopsOpOp2LopsBS.put(OpOp2.AND, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.AND);
- HopsOpOp2LopsBS.put(OpOp2.OR, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.OR);
- HopsOpOp2LopsBS.put(OpOp2.LOG, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.LOG);
- HopsOpOp2LopsBS.put(OpOp2.POW, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.POW);
- HopsOpOp2LopsBS.put(OpOp2.PRINT, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.PRINT);
- HopsOpOp2LopsBS.put(OpOp2.SEQINCR, com.ibm.bi.dml.lops.BinaryScalar.OperationTypes.SEQINCR);
- }
-
- protected static final HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Unary.OperationTypes> HopsOpOp2LopsU;
+ HopsOpOp2LopsBS = new HashMap<Hop.OpOp2, org.apache.sysml.lops.BinaryScalar.OperationTypes>();
+ HopsOpOp2LopsBS.put(OpOp2.PLUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.ADD);
+ HopsOpOp2LopsBS.put(OpOp2.MINUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.SUBTRACT);
+ HopsOpOp2LopsBS.put(OpOp2.MULT, org.apache.sysml.lops.BinaryScalar.OperationTypes.MULTIPLY);
+ HopsOpOp2LopsBS.put(OpOp2.DIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.DIVIDE);
+ HopsOpOp2LopsBS.put(OpOp2.MODULUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.MODULUS);
+ HopsOpOp2LopsBS.put(OpOp2.INTDIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.INTDIV);
+ HopsOpOp2LopsBS.put(OpOp2.LESS, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN);
+ HopsOpOp2LopsBS.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN_OR_EQUALS);
+ HopsOpOp2LopsBS.put(OpOp2.GREATER, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN);
+ HopsOpOp2LopsBS.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN_OR_EQUALS);
+ HopsOpOp2LopsBS.put(OpOp2.EQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.EQUALS);
+ HopsOpOp2LopsBS.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.NOT_EQUALS);
+ HopsOpOp2LopsBS.put(OpOp2.MIN, org.apache.sysml.lops.BinaryScalar.OperationTypes.MIN);
+ HopsOpOp2LopsBS.put(OpOp2.MAX, org.apache.sysml.lops.BinaryScalar.OperationTypes.MAX);
+ HopsOpOp2LopsBS.put(OpOp2.AND, org.apache.sysml.lops.BinaryScalar.OperationTypes.AND);
+ HopsOpOp2LopsBS.put(OpOp2.OR, org.apache.sysml.lops.BinaryScalar.OperationTypes.OR);
+ HopsOpOp2LopsBS.put(OpOp2.LOG, org.apache.sysml.lops.BinaryScalar.OperationTypes.LOG);
+ HopsOpOp2LopsBS.put(OpOp2.POW, org.apache.sysml.lops.BinaryScalar.OperationTypes.POW);
+ HopsOpOp2LopsBS.put(OpOp2.PRINT, org.apache.sysml.lops.BinaryScalar.OperationTypes.PRINT);
+ HopsOpOp2LopsBS.put(OpOp2.SEQINCR, org.apache.sysml.lops.BinaryScalar.OperationTypes.SEQINCR);
+ }
+
+ protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp2LopsU;
static {
- HopsOpOp2LopsU = new HashMap<Hop.OpOp2, com.ibm.bi.dml.lops.Unary.OperationTypes>();
- HopsOpOp2LopsU.put(OpOp2.PLUS, com.ibm.bi.dml.lops.Unary.OperationTypes.ADD);
- HopsOpOp2LopsU.put(OpOp2.MINUS, com.ibm.bi.dml.lops.Unary.OperationTypes.SUBTRACT);
- HopsOpOp2LopsU.put(OpOp2.MULT, com.ibm.bi.dml.lops.Unary.OperationTypes.MULTIPLY);
- HopsOpOp2LopsU.put(OpOp2.DIV, com.ibm.bi.dml.lops.Unary.OperationTypes.DIVIDE);
- HopsOpOp2LopsU.put(OpOp2.MODULUS, com.ibm.bi.dml.lops.Unary.OperationTypes.MODULUS);
- HopsOpOp2LopsU.put(OpOp2.INTDIV, com.ibm.bi.dml.lops.Unary.OperationTypes.INTDIV);
- HopsOpOp2LopsU.put(OpOp2.MINUS1_MULT, com.ibm.bi.dml.lops.Unary.OperationTypes.MINUS1_MULTIPLY);
- HopsOpOp2LopsU.put(OpOp2.LESSEQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.LESS_THAN_OR_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.LESS, com.ibm.bi.dml.lops.Unary.OperationTypes.LESS_THAN);
- HopsOpOp2LopsU.put(OpOp2.GREATEREQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.GREATER_THAN_OR_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.GREATER, com.ibm.bi.dml.lops.Unary.OperationTypes.GREATER_THAN);
- HopsOpOp2LopsU.put(OpOp2.EQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.EQUALS);
- HopsOpOp2LopsU.put(OpOp2.NOTEQUAL, com.ibm.bi.dml.lops.Unary.OperationTypes.NOT_EQUALS);
- HopsOpOp2LopsU.put(OpOp2.AND, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp2LopsU.put(OpOp2.OR, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp2LopsU.put(OpOp2.MAX, com.ibm.bi.dml.lops.Unary.OperationTypes.MAX);
- HopsOpOp2LopsU.put(OpOp2.MIN, com.ibm.bi.dml.lops.Unary.OperationTypes.MIN);
- HopsOpOp2LopsU.put(OpOp2.LOG, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG);
- HopsOpOp2LopsU.put(OpOp2.POW, com.ibm.bi.dml.lops.Unary.OperationTypes.POW);
- HopsOpOp2LopsU.put(OpOp2.MINUS_NZ, com.ibm.bi.dml.lops.Unary.OperationTypes.SUBTRACT_NZ);
- HopsOpOp2LopsU.put(OpOp2.LOG_NZ, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG_NZ);
- }
-
- protected static final HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.Unary.OperationTypes> HopsOpOp1LopsU;
+ HopsOpOp2LopsU = new HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes>();
+ HopsOpOp2LopsU.put(OpOp2.PLUS, org.apache.sysml.lops.Unary.OperationTypes.ADD);
+ HopsOpOp2LopsU.put(OpOp2.MINUS, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT);
+ HopsOpOp2LopsU.put(OpOp2.MULT, org.apache.sysml.lops.Unary.OperationTypes.MULTIPLY);
+ HopsOpOp2LopsU.put(OpOp2.DIV, org.apache.sysml.lops.Unary.OperationTypes.DIVIDE);
+ HopsOpOp2LopsU.put(OpOp2.MODULUS, org.apache.sysml.lops.Unary.OperationTypes.MODULUS);
+ HopsOpOp2LopsU.put(OpOp2.INTDIV, org.apache.sysml.lops.Unary.OperationTypes.INTDIV);
+ HopsOpOp2LopsU.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Unary.OperationTypes.MINUS1_MULTIPLY);
+ HopsOpOp2LopsU.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN_OR_EQUALS);
+ HopsOpOp2LopsU.put(OpOp2.LESS, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN);
+ HopsOpOp2LopsU.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN_OR_EQUALS);
+ HopsOpOp2LopsU.put(OpOp2.GREATER, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN);
+ HopsOpOp2LopsU.put(OpOp2.EQUAL, org.apache.sysml.lops.Unary.OperationTypes.EQUALS);
+ HopsOpOp2LopsU.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Unary.OperationTypes.NOT_EQUALS);
+ HopsOpOp2LopsU.put(OpOp2.AND, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
+ HopsOpOp2LopsU.put(OpOp2.OR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
+ HopsOpOp2LopsU.put(OpOp2.MAX, org.apache.sysml.lops.Unary.OperationTypes.MAX);
+ HopsOpOp2LopsU.put(OpOp2.MIN, org.apache.sysml.lops.Unary.OperationTypes.MIN);
+ HopsOpOp2LopsU.put(OpOp2.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG);
+ HopsOpOp2LopsU.put(OpOp2.POW, org.apache.sysml.lops.Unary.OperationTypes.POW);
+ HopsOpOp2LopsU.put(OpOp2.MINUS_NZ, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT_NZ);
+ HopsOpOp2LopsU.put(OpOp2.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ);
+ }
+
+ protected static final HashMap<Hop.OpOp1, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp1LopsU;
static {
- HopsOpOp1LopsU = new HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.Unary.OperationTypes>();
- HopsOpOp1LopsU.put(OpOp1.NOT, com.ibm.bi.dml.lops.Unary.OperationTypes.NOT);
- HopsOpOp1LopsU.put(OpOp1.ABS, com.ibm.bi.dml.lops.Unary.OperationTypes.ABS);
- HopsOpOp1LopsU.put(OpOp1.SIN, com.ibm.bi.dml.lops.Unary.OperationTypes.SIN);
- HopsOpOp1LopsU.put(OpOp1.COS, com.ibm.bi.dml.lops.Unary.OperationTypes.COS);
- HopsOpOp1LopsU.put(OpOp1.TAN, com.ibm.bi.dml.lops.Unary.OperationTypes.TAN);
- HopsOpOp1LopsU.put(OpOp1.ASIN, com.ibm.bi.dml.lops.Unary.OperationTypes.ASIN);
- HopsOpOp1LopsU.put(OpOp1.ACOS, com.ibm.bi.dml.lops.Unary.OperationTypes.ACOS);
- HopsOpOp1LopsU.put(OpOp1.ATAN, com.ibm.bi.dml.lops.Unary.OperationTypes.ATAN);
- HopsOpOp1LopsU.put(OpOp1.SQRT, com.ibm.bi.dml.lops.Unary.OperationTypes.SQRT);
- HopsOpOp1LopsU.put(OpOp1.EXP, com.ibm.bi.dml.lops.Unary.OperationTypes.EXP);
- HopsOpOp1LopsU.put(OpOp1.LOG, com.ibm.bi.dml.lops.Unary.OperationTypes.LOG);
- HopsOpOp1LopsU.put(OpOp1.ROUND, com.ibm.bi.dml.lops.Unary.OperationTypes.ROUND);
- HopsOpOp1LopsU.put(OpOp1.CEIL, com.ibm.bi.dml.lops.Unary.OperationTypes.CEIL);
- HopsOpOp1LopsU.put(OpOp1.FLOOR, com.ibm.bi.dml.lops.Unary.OperationTypes.FLOOR);
- HopsOpOp1LopsU.put(OpOp1.CUMSUM, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMSUM);
- HopsOpOp1LopsU.put(OpOp1.CUMPROD, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMPROD);
- HopsOpOp1LopsU.put(OpOp1.CUMMIN, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMMIN);
- HopsOpOp1LopsU.put(OpOp1.CUMMAX, com.ibm.bi.dml.lops.Unary.OperationTypes.CUMMAX);
- HopsOpOp1LopsU.put(OpOp1.INVERSE, com.ibm.bi.dml.lops.Unary.OperationTypes.INVERSE);
- HopsOpOp1LopsU.put(OpOp1.CAST_AS_SCALAR, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, com.ibm.bi.dml.lops.Unary.OperationTypes.NOTSUPPORTED);
- HopsOpOp1LopsU.put(OpOp1.SPROP, com.ibm.bi.dml.lops.Unary.OperationTypes.SPROP);
- HopsOpOp1LopsU.put(OpOp1.SIGMOID, com.ibm.bi.dml.lops.Unary.OperationTypes.SIGMOID);
- HopsOpOp1LopsU.put(OpOp1.SELP, com.ibm.bi.dml.lops.Unary.OperationTypes.SELP);
- }
-
- protected static final HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.UnaryCP.OperationTypes> HopsOpOp1LopsUS;
+ HopsOpOp1LopsU = new HashMap<Hop.OpOp1, org.apache.sysml.lops.Unary.OperationTypes>();
+ HopsOpOp1LopsU.put(OpOp1.NOT, org.apache.sysml.lops.Unary.OperationTypes.NOT);
+ HopsOpOp1LopsU.put(OpOp1.ABS, org.apache.sysml.lops.Unary.OperationTypes.ABS);
+ HopsOpOp1LopsU.put(OpOp1.SIN, org.apache.sysml.lops.Unary.OperationTypes.SIN);
+ HopsOpOp1LopsU.put(OpOp1.COS, org.apache.sysml.lops.Unary.OperationTypes.COS);
+ HopsOpOp1LopsU.put(OpOp1.TAN, org.apache.sysml.lops.Unary.OperationTypes.TAN);
+ HopsOpOp1LopsU.put(OpOp1.ASIN, org.apache.sysml.lops.Unary.OperationTypes.ASIN);
+ HopsOpOp1LopsU.put(OpOp1.ACOS, org.apache.sysml.lops.Unary.OperationTypes.ACOS);
+ HopsOpOp1LopsU.put(OpOp1.ATAN, org.apache.sysml.lops.Unary.OperationTypes.ATAN);
+ HopsOpOp1LopsU.put(OpOp1.SQRT, org.apache.sysml.lops.Unary.OperationTypes.SQRT);
+ HopsOpOp1LopsU.put(OpOp1.EXP, org.apache.sysml.lops.Unary.OperationTypes.EXP);
+ HopsOpOp1LopsU.put(OpOp1.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG);
+ HopsOpOp1LopsU.put(OpOp1.ROUND, org.apache.sysml.lops.Unary.OperationTypes.ROUND);
+ HopsOpOp1LopsU.put(OpOp1.CEIL, org.apache.sysml.lops.Unary.OperationTypes.CEIL);
+ HopsOpOp1LopsU.put(OpOp1.FLOOR, org.apache.sysml.lops.Unary.OperationTypes.FLOOR);
+ HopsOpOp1LopsU.put(OpOp1.CUMSUM, org.apache.sysml.lops.Unary.OperationTypes.CUMSUM);
+ HopsOpOp1LopsU.put(OpOp1.CUMPROD, org.apache.sysml.lops.Unary.OperationTypes.CUMPROD);
+ HopsOpOp1LopsU.put(OpOp1.CUMMIN, org.apache.sysml.lops.Unary.OperationTypes.CUMMIN);
+ HopsOpOp1LopsU.put(OpOp1.CUMMAX, org.apache.sysml.lops.Unary.OperationTypes.CUMMAX);
+ HopsOpOp1LopsU.put(OpOp1.INVERSE, org.apache.sysml.lops.Unary.OperationTypes.INVERSE);
+ HopsOpOp1LopsU.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
+ HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
+ HopsOpOp1LopsU.put(OpOp1.SPROP, org.apache.sysml.lops.Unary.OperationTypes.SPROP);
+ HopsOpOp1LopsU.put(OpOp1.SIGMOID, org.apache.sysml.lops.Unary.OperationTypes.SIGMOID);
+ HopsOpOp1LopsU.put(OpOp1.SELP, org.apache.sysml.lops.Unary.OperationTypes.SELP);
+ }
+
+ protected static final HashMap<Hop.OpOp1, org.apache.sysml.lops.UnaryCP.OperationTypes> HopsOpOp1LopsUS;
static {
- HopsOpOp1LopsUS = new HashMap<Hop.OpOp1, com.ibm.bi.dml.lops.UnaryCP.OperationTypes>();
- HopsOpOp1LopsUS.put(OpOp1.NOT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NOT);
- HopsOpOp1LopsUS.put(OpOp1.ABS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ABS);
- HopsOpOp1LopsUS.put(OpOp1.SIN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.SIN);
- HopsOpOp1LopsUS.put(OpOp1.COS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.COS);
- HopsOpOp1LopsUS.put(OpOp1.TAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.TAN);
- HopsOpOp1LopsUS.put(OpOp1.ASIN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ASIN);
- HopsOpOp1LopsUS.put(OpOp1.ACOS, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ACOS);
- HopsOpOp1LopsUS.put(OpOp1.ATAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ATAN);
- HopsOpOp1LopsUS.put(OpOp1.SQRT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.SQRT);
- HopsOpOp1LopsUS.put(OpOp1.EXP, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.EXP);
- HopsOpOp1LopsUS.put(OpOp1.LOG, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.LOG);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_INT);
- HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN);
- HopsOpOp1LopsUS.put(OpOp1.NROW, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NROW);
- HopsOpOp1LopsUS.put(OpOp1.NCOL, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.NCOL);
- HopsOpOp1LopsUS.put(OpOp1.LENGTH, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.LENGTH);
- HopsOpOp1LopsUS.put(OpOp1.PRINT, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.PRINT);
- HopsOpOp1LopsUS.put(OpOp1.ROUND, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.ROUND);
- HopsOpOp1LopsUS.put(OpOp1.CEIL, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.CEIL);
- HopsOpOp1LopsUS.put(OpOp1.FLOOR, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.FLOOR);
- HopsOpOp1LopsUS.put(OpOp1.STOP, com.ibm.bi.dml.lops.UnaryCP.OperationTypes.STOP);
+ HopsOpOp1LopsUS = new HashMap<Hop.OpOp1, org.apache.sysml.lops.UnaryCP.OperationTypes>();
+ HopsOpOp1LopsUS.put(OpOp1.NOT, org.apache.sysml.lops.UnaryCP.OperationTypes.NOT);
+ HopsOpOp1LopsUS.put(OpOp1.ABS, org.apache.sysml.lops.UnaryCP.OperationTypes.ABS);
+ HopsOpOp1LopsUS.put(OpOp1.SIN, org.apache.sysml.lops.UnaryCP.OperationTypes.SIN);
+ HopsOpOp1LopsUS.put(OpOp1.COS, org.apache.sysml.lops.UnaryCP.OperationTypes.COS);
+ HopsOpOp1LopsUS.put(OpOp1.TAN, org.apache.sysml.lops.UnaryCP.OperationTypes.TAN);
+ HopsOpOp1LopsUS.put(OpOp1.ASIN, org.apache.sysml.lops.UnaryCP.OperationTypes.ASIN);
+ HopsOpOp1LopsUS.put(OpOp1.ACOS, org.apache.sysml.lops.UnaryCP.OperationTypes.ACOS);
+ HopsOpOp1LopsUS.put(OpOp1.ATAN, org.apache.sysml.lops.UnaryCP.OperationTypes.ATAN);
+ HopsOpOp1LopsUS.put(OpOp1.SQRT, org.apache.sysml.lops.UnaryCP.OperationTypes.SQRT);
+ HopsOpOp1LopsUS.put(OpOp1.EXP, org.apache.sysml.lops.UnaryCP.OperationTypes.EXP);
+ HopsOpOp1LopsUS.put(OpOp1.LOG, org.apache.sysml.lops.UnaryCP.OperationTypes.LOG);
+ HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR);
+ HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX);
+ HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE);
+ HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_INT);
+ HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN);
+ HopsOpOp1LopsUS.put(OpOp1.NROW, org.apache.sysml.lops.UnaryCP.OperationTypes.NROW);
+ HopsOpOp1LopsUS.put(OpOp1.NCOL, org.apache.sysml.lops.UnaryCP.OperationTypes.NCOL);
+ HopsOpOp1LopsUS.put(OpOp1.LENGTH, org.apache.sysml.lops.UnaryCP.OperationTypes.LENGTH);
+ HopsOpOp1LopsUS.put(OpOp1.PRINT, org.apache.sysml.lops.UnaryCP.OperationTypes.PRINT);
+ HopsOpOp1LopsUS.put(OpOp1.ROUND, org.apache.sysml.lops.UnaryCP.OperationTypes.ROUND);
+ HopsOpOp1LopsUS.put(OpOp1.CEIL, org.apache.sysml.lops.UnaryCP.OperationTypes.CEIL);
+ HopsOpOp1LopsUS.put(OpOp1.FLOOR, org.apache.sysml.lops.UnaryCP.OperationTypes.FLOOR);
+ HopsOpOp1LopsUS.put(OpOp1.STOP, org.apache.sysml.lops.UnaryCP.OperationTypes.STOP);
}
protected static final HashMap<Hop.OpOp1, String> HopsOpOp12String;
@@ -1305,15 +1305,15 @@ public abstract class Hop
HopsOpOp12String.put(OpOp1.SIGMOID, "sigmoid");
}
- protected static final HashMap<Hop.ParamBuiltinOp, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes> HopsParameterizedBuiltinLops;
+ protected static final HashMap<Hop.ParamBuiltinOp, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes> HopsParameterizedBuiltinLops;
static {
- HopsParameterizedBuiltinLops = new HashMap<Hop.ParamBuiltinOp, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes>();
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.CDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.INVCDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORM, com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM);
+ HopsParameterizedBuiltinLops = new HashMap<Hop.ParamBuiltinOp, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes>();
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.CDF);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.INVCDF);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORM, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM);
}
protected static final HashMap<Hop.OpOp2, String> HopsOpOp2String;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/HopsException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/HopsException.java b/src/main/java/org/apache/sysml/hops/HopsException.java
index d1c4901..2fb1275 100644
--- a/src/main/java/org/apache/sysml/hops/HopsException.java
+++ b/src/main/java/org/apache/sysml/hops/HopsException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
/**
* <p>Exception occurring in the HOP level.</p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index b351d61..9ef0607 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RangeBasedReIndex;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.RangeBasedReIndex;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
//for now only works for range based indexing op
public class IndexingOp extends Hop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index d22cf67..77ffaa4 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.LeftIndex;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RangeBasedReIndex;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.ZeroOut;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.UnaryCP.OperationTypes;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.lops.Binary;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.RangeBasedReIndex;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.ZeroOut;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.UnaryCP.OperationTypes;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
public class LeftIndexingOp extends Hop
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/LiteralOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LiteralOp.java b/src/main/java/org/apache/sysml/hops/LiteralOp.java
index 0ed419d..3683a26 100644
--- a/src/main/java/org/apache/sysml/hops/LiteralOp.java
+++ b/src/main/java/org/apache/sysml/hops/LiteralOp.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class LiteralOp extends Hop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/MemoTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/MemoTable.java b/src/main/java/org/apache/sysml/hops/MemoTable.java
index 428d3e9..e8e2e5c 100644
--- a/src/main/java/org/apache/sysml/hops/MemoTable.java
+++ b/src/main/java/org/apache/sysml/hops/MemoTable.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.recompile.RecompileStatus;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.recompile.RecompileStatus;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/**
* Memoization Table (hop id, worst-case matrix characteristics).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 136abd0..3564220 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -15,34 +15,34 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
public class OptimizerUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 8387736..e0653aa 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.GroupedAggregate;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.GroupedAggregate;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.OutputParameters.Format;
+import org.apache.sysml.lops.ParameterizedBuiltin;
+import org.apache.sysml.lops.RepMat;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
index fd329f0..4bb2c8c 100644
--- a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
@@ -15,41 +15,41 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSigmoidR;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.DataPartition;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.RepMat;
+import org.apache.sysml.lops.Transform;
+import org.apache.sysml.lops.Unary;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedCrossEntropyR;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSigmoidR;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
/**
* Note: this hop should be called AggQuaternaryOp in consistency with AggUnaryOp and AggBinaryOp;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/ReorgOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java
index 703fb4a..fb2679b 100644
--- a/src/main/java/org/apache/sysml/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.lops.Transform;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/**
* Reorg (cell) operation: aij
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/TernaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/TernaryOp.java b/src/main/java/org/apache/sysml/hops/TernaryOp.java
index c1cb96b..33270c0 100644
--- a/src/main/java/org/apache/sysml/hops/TernaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/TernaryOp.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.CentralMoment;
-import com.ibm.bi.dml.lops.CoVariance;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.CombineTernary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Ternary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.CombineBinary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.CentralMoment;
+import org.apache.sysml.lops.CoVariance;
+import org.apache.sysml.lops.CombineBinary;
+import org.apache.sysml.lops.CombineTernary;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.PickByCount;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.lops.Ternary;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.CombineBinary.OperationTypes;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/* Primary use cases for now, are
* quantile (<n-1-matrix>, <n-1-matrix>, <literal>): quantile (A, w, 0.5)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 71af54f..e770425 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -15,30 +15,30 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.CombineUnary;
-import com.ibm.bi.dml.lops.CumulativeOffsetBinary;
-import com.ibm.bi.dml.lops.CumulativePartialAggregate;
-import com.ibm.bi.dml.lops.CumulativeSplitAggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Aggregate.OperationTypes;
+import org.apache.sysml.lops.CombineUnary;
+import org.apache.sysml.lops.CumulativeOffsetBinary;
+import org.apache.sysml.lops.CumulativePartialAggregate;
+import org.apache.sysml.lops.CumulativeSplitAggregate;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.PartialAggregate;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.PickByCount;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.lops.Unary;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/* Unary (cell operations): e.g, b_ij = round(a_ij)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
index b4c669b..024fe95 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.cost;
+package org.apache.sysml.hops.cost;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,16 +26,16 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
public class CostEstimationWrapper
{
@@ -57,7 +57,7 @@ public class CostEstimationWrapper
{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.cost")
+ Logger.getLogger("org.apache.sysml.hops.cost")
.setLevel((Level) Level.DEBUG);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 1f6d947..3349892 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.cost;
+package org.apache.sysml.hops.cost;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,47 +27,47 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateTernaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateUnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DataGenCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MMTSJCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.StringInitCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.UnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.AggregateTernaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.AggregateUnaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MMTSJCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.StringInitCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.UnaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public abstract class CostEstimator
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorNumMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorNumMRJobs.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorNumMRJobs.java
index 4c9e78f..6191954 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorNumMRJobs.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorNumMRJobs.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.hops.cost;
+package org.apache.sysml.hops.cost;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
index 67d9cb2..ff360c1 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
@@ -15,53 +15,53 @@
*
*/
-package com.ibm.bi.dml.hops.cost;
+package org.apache.sysml.hops.cost;
import java.util.ArrayList;
import java.util.HashSet;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.LazyWriteBuffer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.IDistributedCacheConsumer;
-import com.ibm.bi.dml.runtime.instructions.mr.MMTSJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MapMultChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PickByCountInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.yarn.ropt.MRJobResourceInstruction;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.IDistributedCacheConsumer;
+import org.apache.sysml.runtime.instructions.mr.MMTSJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MapMultChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.PickByCountInstruction;
+import org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.TernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.yarn.ropt.MRJobResourceInstruction;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/cost/VarStats.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/VarStats.java b/src/main/java/org/apache/sysml/hops/cost/VarStats.java
index b1d9671..ebee6b7 100644
--- a/src/main/java/org/apache/sysml/hops/cost/VarStats.java
+++ b/src/main/java/org/apache/sysml/hops/cost/VarStats.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.cost;
+package org.apache.sysml.hops.cost;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
index d5d005b..b9272a7 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,34 +23,34 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.cost.CostEstimationWrapper;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFLoopNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.MismatchHeuristicFactory;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.cost.CostEstimationWrapper;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFGraph;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFLoopNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType;
+import org.apache.sysml.hops.globalopt.gdfresolve.GDFMismatchHeuristic;
+import org.apache.sysml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
+import org.apache.sysml.hops.globalopt.gdfresolve.MismatchHeuristicFactory;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
/**
* Global data flow optimization via enumeration-based optimizer (dynamic programming).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizer.java b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizer.java
index f48ad33..bc5ed6a 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizer.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizer.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFGraph;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
/**
* Super class for all optimizers (e.g., transformation-based, and enumeration-based)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizerWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizerWrapper.java b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizerWrapper.java
index be15d07..dc86e61 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizerWrapper.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/GlobalOptimizerWrapper.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFGraph;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GraphBuilder;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.utils.Explain;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFGraph;
+import org.apache.sysml.hops.globalopt.gdfgraph.GraphBuilder;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.utils.Explain;
/**
* Main entry point for Global Data Flow Optimization. It is intended to be invoked after
@@ -58,7 +58,7 @@ public class GlobalOptimizerWrapper
{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.globalopt")
+ Logger.getLogger("org.apache.sysml.hops.globalopt")
.setLevel((Level) Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java b/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java
index 7f58b09..acc196e 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import com.google.common.base.Objects;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/MemoStructure.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/MemoStructure.java b/src/main/java/org/apache/sysml/hops/globalopt/MemoStructure.java
index 0cdd576..b35e313 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/MemoStructure.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/MemoStructure.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
/**
* This MemoStructure is the central location for storing enumerated plans (configurations) and serves
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/Plan.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/Plan.java b/src/main/java/org/apache/sysml/hops/globalopt/Plan.java
index aa9b29e..1e772ac 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/Plan.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/Plan.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
public class Plan
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/PlanSet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/PlanSet.java b/src/main/java/org/apache/sysml/hops/globalopt/PlanSet.java
index cf1897d..276657b 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/PlanSet.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/PlanSet.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFCrossBlockNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode;
-import com.ibm.bi.dml.hops.globalopt.gdfgraph.GDFNode.NodeType;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFCrossBlockNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
+import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType;
public class PlanSet
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/RewriteConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/RewriteConfig.java b/src/main/java/org/apache/sysml/hops/globalopt/RewriteConfig.java
index a202a8a..417cd43 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/RewriteConfig.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/RewriteConfig.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties.Location;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.globalopt.InterestingProperties.Location;
+import org.apache.sysml.lops.LopProperties.ExecType;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/Summary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/Summary.java b/src/main/java/org/apache/sysml/hops/globalopt/Summary.java
index 2814a61..361dd80 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/Summary.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/Summary.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt;
+package org.apache.sysml.hops.globalopt;
public class Summary
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
index 60d95ce..232698d 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFCrossBlockNode.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
+package org.apache.sysml.hops.globalopt.gdfgraph;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
/**
* Crossblock operators represent
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFGraph.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFGraph.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFGraph.java
index 99b177d..d2ccc4d 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFGraph.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFGraph.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
+package org.apache.sysml.hops.globalopt.gdfgraph;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.Program;
public class GDFGraph
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFLoopNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFLoopNode.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFLoopNode.java
index 55c3ed1..4d71405 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFLoopNode.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFLoopNode.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
+package org.apache.sysml.hops.globalopt.gdfgraph;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public class GDFLoopNode extends GDFNode
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java
index 8a048cd..0de53b4 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
+package org.apache.sysml.hops.globalopt.gdfgraph;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
/**
* The reason of a custom graph structure is to unify both within DAG
[33/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteUtils.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteUtils.java
deleted file mode 100644
index 6da131c..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteUtils.java
+++ /dev/null
@@ -1,1071 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-public class HopRewriteUtils
-{
-
-
- /**
- *
- * @param op
- * @return
- */
- public static boolean isValueTypeCast( OpOp1 op )
- {
- return ( op == OpOp1.CAST_AS_BOOLEAN
- || op == OpOp1.CAST_AS_INT
- || op == OpOp1.CAST_AS_DOUBLE );
- }
-
- //////////////////////////////////
- // literal handling
-
- /**
- *
- * @param op
- * @return
- * @throws HopsException
- */
- public static boolean getBooleanValue( LiteralOp op )
- throws HopsException
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return op.getDoubleValue() != 0;
- case INT: return op.getLongValue() != 0;
- case BOOLEAN: return op.getBooleanValue();
-
- default: throw new HopsException("Invalid boolean value: "+op.getValueType());
- }
- }
-
- public static boolean getBooleanValueSafe( LiteralOp op )
- {
- try
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return op.getDoubleValue() != 0;
- case INT: return op.getLongValue() != 0;
- case BOOLEAN: return op.getBooleanValue();
-
- default: throw new HopsException("Invalid boolean value: "+op.getValueType());
- }
- }
- catch(Exception ex){
- //silently ignore error
- }
-
- return false;
- }
-
- /**
- *
- * @param op
- * @return
- * @throws HopsException
- */
- public static double getDoubleValue( LiteralOp op )
- throws HopsException
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return op.getDoubleValue();
- case INT: return op.getLongValue();
- case BOOLEAN: return op.getBooleanValue() ? 1 : 0;
-
- default: throw new HopsException("Invalid double value: "+op.getValueType());
- }
- }
-
- /**
- *
- * @param op
- * @return
- */
- public static double getDoubleValueSafe( LiteralOp op )
- {
- try
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return op.getDoubleValue();
- case INT: return op.getLongValue();
- case BOOLEAN: return op.getBooleanValue() ? 1 : 0;
-
- default: throw new HopsException("Invalid double value: "+op.getValueType());
- }
- }
- catch(Exception ex){
- //silently ignore error
- }
-
- return Double.MAX_VALUE;
- }
-
- /**
- * Return the int value of a LiteralOp (as a long).
- *
- * Note: For comparisons, this is *only* to be used in situations
- * in which the value is absolutely guaranteed to be an integer.
- * Otherwise, a safer alternative is `getDoubleValue`.
- */
- public static long getIntValue( LiteralOp op )
- throws HopsException
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return UtilFunctions.toLong(op.getDoubleValue());
- case INT: return op.getLongValue();
- case BOOLEAN: return op.getBooleanValue() ? 1 : 0;
-
- default: throw new HopsException("Invalid int value: "+op.getValueType());
- }
- }
-
- /**
- *
- * @param op
- * @return
- */
- public static long getIntValueSafe( LiteralOp op )
- {
- try
- {
- switch( op.getValueType() )
- {
- case DOUBLE: return UtilFunctions.toLong(op.getDoubleValue());
- case INT: return op.getLongValue();
- case BOOLEAN: return op.getBooleanValue() ? 1 : 0;
- default:
- throw new RuntimeException("Invalid int value: "+op.getValueType());
- }
- }
- catch(Exception ex){
- //silently ignore error
- }
-
- return Long.MAX_VALUE;
- }
-
- /**
- *
- * @param op
- * @return
- * @throws HopsException
- */
- public static ScalarObject getScalarObject( LiteralOp op )
- {
- ScalarObject ret = null;
-
- try
- {
- switch( op.getValueType() )
- {
- case DOUBLE: ret = new DoubleObject(op.getDoubleValue()); break;
- case INT: ret = new IntObject(op.getLongValue()); break;
- case BOOLEAN: ret = new BooleanObject(op.getBooleanValue()); break;
- case STRING: ret = new StringObject(op.getStringValue()); break;
- default:
- throw new DMLRuntimeException("Invalid scalar object value type: "+op.getValueType());
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException("Failed to create scalar object for constant. Continue.", ex);
- }
-
- return ret;
- }
-
-
- ///////////////////////////////////
- // hop dag transformations
-
-
-
- public static int getChildReferencePos( Hop parent, Hop child )
- {
- ArrayList<Hop> childs = parent.getInput();
- return childs.indexOf(child);
- }
-
- public static void removeChildReference( Hop parent, Hop child )
- {
- //remove child reference
- parent.getInput().remove( child );
- child.getParent().remove( parent );
- }
-
- public static void removeChildReferenceByPos( Hop parent, Hop child, int posChild )
- {
- //remove child reference
- parent.getInput().remove( posChild );
- child.getParent().remove( parent );
- }
-
- public static void removeChildReferenceByPos( Hop parent, Hop child, int posChild, int posParent )
- {
- //remove child reference
- parent.getInput().remove( posChild );
- child.getParent().remove( posParent );
- }
-
- public static void removeAllChildReferences( Hop parent )
- {
- //remove parent reference from all childs
- for( Hop child : parent.getInput() )
- child.getParent().remove(parent);
-
- //remove all child references
- parent.getInput().clear();
- }
-
- public static void addChildReference( Hop parent, Hop child )
- {
- parent.getInput().add( child );
- child.getParent().add( parent );
- }
-
- public static void addChildReference( Hop parent, Hop child, int pos )
- {
- parent.getInput().add( pos, child );
- child.getParent().add( parent );
- }
-
- /**
- *
- * @param input
- * @param value
- * @return
- * @throws HopsException
- */
- public static Hop createDataGenOp( Hop input, double value )
- throws HopsException
- {
- Hop rows = (input.getDim1()>0) ? new LiteralOp(input.getDim1()) :
- new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input);
- Hop cols = (input.getDim2()>0) ? new LiteralOp(input.getDim2()) :
- new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, input);
- Hop val = new LiteralOp(value);
-
- HashMap<String, Hop> params = new HashMap<String, Hop>();
- params.put(DataExpression.RAND_ROWS, rows);
- params.put(DataExpression.RAND_COLS, cols);
- params.put(DataExpression.RAND_MIN, val);
- params.put(DataExpression.RAND_MAX, val);
- params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
- params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0));
- params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
- params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) );
-
- //note internal refresh size information
- Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
- datagen.setRowsInBlock(input.getRowsInBlock());
- datagen.setColsInBlock(input.getColsInBlock());
-
- if( value==0 )
- datagen.setNnz(0);
-
- return datagen;
- }
-
- /**
- * Assumes that min and max are literal ops, needs to be checked from outside.
- *
- * @param inputGen
- * @param scale
- * @param intercept
- * @return
- * @throws HopsException
- */
- public static DataGenOp copyDataGenOp( DataGenOp inputGen, double scale, double shift )
- throws HopsException
- {
- HashMap<String, Integer> params = inputGen.getParamIndexMap();
- Hop rows = inputGen.getInput().get(params.get(DataExpression.RAND_ROWS));
- Hop cols = inputGen.getInput().get(params.get(DataExpression.RAND_COLS));
- Hop min = inputGen.getInput().get(params.get(DataExpression.RAND_MIN));
- Hop max = inputGen.getInput().get(params.get(DataExpression.RAND_MAX));
- Hop pdf = inputGen.getInput().get(params.get(DataExpression.RAND_PDF));
- Hop mean = inputGen.getInput().get(params.get(DataExpression.RAND_LAMBDA));
- Hop sparsity = inputGen.getInput().get(params.get(DataExpression.RAND_SPARSITY));
- Hop seed = inputGen.getInput().get(params.get(DataExpression.RAND_SEED));
-
- //check for literal ops
- if( !(min instanceof LiteralOp) || !(max instanceof LiteralOp))
- return null;
-
- //scale and shift
- double smin = getDoubleValue((LiteralOp) min);
- double smax = getDoubleValue((LiteralOp) max);
- smin = smin * scale + shift;
- smax = smax * scale + shift;
-
- Hop sminHop = new LiteralOp(smin);
- Hop smaxHop = new LiteralOp(smax);
-
- HashMap<String, Hop> params2 = new HashMap<String, Hop>();
- params2.put(DataExpression.RAND_ROWS, rows);
- params2.put(DataExpression.RAND_COLS, cols);
- params2.put(DataExpression.RAND_MIN, sminHop);
- params2.put(DataExpression.RAND_MAX, smaxHop);
- params2.put(DataExpression.RAND_PDF, pdf);
- params2.put(DataExpression.RAND_LAMBDA, mean);
- params2.put(DataExpression.RAND_SPARSITY, sparsity);
- params2.put(DataExpression.RAND_SEED, seed );
-
- //note internal refresh size information
- DataGenOp datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params2);
- datagen.setRowsInBlock(inputGen.getRowsInBlock());
- datagen.setColsInBlock(inputGen.getColsInBlock());
-
- if( smin==0 && smax==0 ) {
- datagen.setNnz(0);
- }
-
- return datagen;
- }
-
- public static Hop createDataGenOp( Hop rowInput, Hop colInput, double value )
- throws HopsException
- {
- Hop rows = (rowInput.getDim1()>0) ? new LiteralOp(rowInput.getDim1()) :
- new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, rowInput);
- Hop cols = (colInput.getDim2()>0) ? new LiteralOp(colInput.getDim2()) :
- new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, colInput);
- Hop val = new LiteralOp(value);
-
- HashMap<String, Hop> params = new HashMap<String, Hop>();
- params.put(DataExpression.RAND_ROWS, rows);
- params.put(DataExpression.RAND_COLS, cols);
- params.put(DataExpression.RAND_MIN, val);
- params.put(DataExpression.RAND_MAX, val);
- params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
- params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0));
- params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
- params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) );
-
- //note internal refresh size information
- Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
- datagen.setRowsInBlock(rowInput.getRowsInBlock());
- datagen.setColsInBlock(colInput.getColsInBlock());
-
- if( value==0 )
- datagen.setNnz(0);
-
- return datagen;
- }
-
- public static Hop createDataGenOp( Hop rowInput, boolean tRowInput, Hop colInput, boolean tColInput, double value )
- throws HopsException
- {
- long nrow = tRowInput ? rowInput.getDim2() : rowInput.getDim1();
- long ncol = tColInput ? colInput.getDim1() : rowInput.getDim2();
-
- Hop rows = (nrow>0) ? new LiteralOp(nrow) :
- new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, tRowInput?OpOp1.NCOL:OpOp1.NROW, rowInput);
- Hop cols = (ncol>0) ? new LiteralOp(ncol) :
- new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, tColInput?OpOp1.NROW:OpOp1.NCOL, colInput);
- Hop val = new LiteralOp(value);
-
- HashMap<String, Hop> params = new HashMap<String, Hop>();
- params.put(DataExpression.RAND_ROWS, rows);
- params.put(DataExpression.RAND_COLS, cols);
- params.put(DataExpression.RAND_MIN, val);
- params.put(DataExpression.RAND_MAX, val);
- params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
- params.put(DataExpression.RAND_LAMBDA,new LiteralOp(-1.0));
- params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
- params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) );
-
- //note internal refresh size information
- Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
- datagen.setRowsInBlock(rowInput.getRowsInBlock());
- datagen.setColsInBlock(colInput.getColsInBlock());
-
- if( value==0 )
- datagen.setNnz(0);
-
- return datagen;
- }
-
- public static Hop createDataGenOpByVal( Hop rowInput, Hop colInput, double value )
- throws HopsException
- {
- Hop val = new LiteralOp(value);
-
- HashMap<String, Hop> params = new HashMap<String, Hop>();
- params.put(DataExpression.RAND_ROWS, rowInput);
- params.put(DataExpression.RAND_COLS, colInput);
- params.put(DataExpression.RAND_MIN, val);
- params.put(DataExpression.RAND_MAX, val);
- params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
- params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0));
- params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
- params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) );
-
- //note internal refresh size information
- Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
- datagen.setRowsInBlock(rowInput.getRowsInBlock());
- datagen.setColsInBlock(colInput.getColsInBlock());
-
- if( value==0 )
- datagen.setNnz(0);
-
- return datagen;
- }
-
- /**
- *
- * @param input
- * @return
- */
- public static ReorgOp createTranspose(Hop input)
- {
- ReorgOp transpose = new ReorgOp(input.getName(), input.getDataType(), input.getValueType(), ReOrgOp.TRANSPOSE, input);
- HopRewriteUtils.setOutputBlocksizes(transpose, input.getRowsInBlock(), input.getColsInBlock());
- HopRewriteUtils.copyLineNumbers(input, transpose);
- transpose.refreshSizeInformation();
-
- return transpose;
- }
-
- /**
- *
- * @param input
- * @param type
- * @return
- * @throws HopsException
- */
- public static UnaryOp createUnary(Hop input, OpOp1 type)
- throws HopsException
- {
- UnaryOp unary = new UnaryOp(input.getName(), input.getDataType(), input.getValueType(), type, input);
- HopRewriteUtils.setOutputBlocksizes(unary, input.getRowsInBlock(), input.getColsInBlock());
- HopRewriteUtils.copyLineNumbers(input, unary);
- unary.refreshSizeInformation();
-
- return unary;
- }
-
- /**
- *
- * @param input
- * @return
- */
- public static BinaryOp createMinus(Hop input)
- {
- BinaryOp minus = new BinaryOp(input.getName(), input.getDataType(), input.getValueType(), OpOp2.MINUS, new LiteralOp(0), input);
- HopRewriteUtils.setOutputBlocksizes(minus, input.getRowsInBlock(), input.getColsInBlock());
- HopRewriteUtils.copyLineNumbers(input, minus);
- minus.refreshSizeInformation();
-
- return minus;
- }
-
- /**
- *
- * @param input1
- * @param input2
- * @param op
- * @return
- */
- public static BinaryOp createBinary(Hop input1, Hop input2, OpOp2 op)
- {
- BinaryOp bop = new BinaryOp(input1.getName(), input1.getDataType(),
- input1.getValueType(), op, input1, input2);
- HopRewriteUtils.setOutputBlocksizes(bop, input1.getRowsInBlock(), input1.getColsInBlock());
- HopRewriteUtils.copyLineNumbers(input1, bop);
- bop.refreshSizeInformation();
-
- return bop;
- }
-
- /**
- *
- * @param left
- * @param right
- * @return
- */
- public static AggBinaryOp createMatrixMultiply(Hop left, Hop right)
- {
- AggBinaryOp mmult = new AggBinaryOp(left.getName(), left.getDataType(), left.getValueType(), OpOp2.MULT, AggOp.SUM, left, right);
- mmult.setRowsInBlock(left.getRowsInBlock());
- mmult.setColsInBlock(right.getColsInBlock());
- mmult.refreshSizeInformation();
-
- return mmult;
- }
-
- public static Hop createValueHop( Hop hop, boolean row )
- throws HopsException
- {
- Hop ret = null;
- if( row ){
- ret = (hop.getDim1()>0) ? new LiteralOp(hop.getDim1()) :
- new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, hop);
- }
- else{
- ret = (hop.getDim2()>0) ? new LiteralOp(hop.getDim2()) :
- new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, hop);
- }
-
- return ret;
- }
-
-
- public static DataGenOp createSeqDataGenOp( Hop input )
- throws HopsException
- {
- return createSeqDataGenOp(input, true);
- }
-
- public static DataGenOp createSeqDataGenOp( Hop input, boolean asc )
- throws HopsException
- {
- Hop to = (input.getDim1()>0) ? new LiteralOp(input.getDim1()) :
- new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input);
-
- HashMap<String, Hop> params = new HashMap<String, Hop>();
- if( asc ) {
- params.put(Statement.SEQ_FROM, new LiteralOp(1));
- params.put(Statement.SEQ_TO, to);
- params.put(Statement.SEQ_INCR, new LiteralOp(1));
- }
- else {
- params.put(Statement.SEQ_FROM, to);
- params.put(Statement.SEQ_TO, new LiteralOp(1));
- params.put(Statement.SEQ_INCR, new LiteralOp(-1));
- }
-
- //note internal refresh size information
- DataGenOp datagen = new DataGenOp(DataGenMethod.SEQ, new DataIdentifier("tmp"), params);
- datagen.setRowsInBlock(input.getRowsInBlock());
- datagen.setColsInBlock(input.getColsInBlock());
-
- return datagen;
- }
-
- public static void setOutputBlocksizes( Hop hop, long brlen, long bclen )
- {
- hop.setRowsInBlock( brlen );
- hop.setColsInBlock( bclen );
- }
-
- public static void setOutputParameters( Hop hop, long rlen, long clen, long brlen, long bclen, long nnz )
- {
- hop.setDim1( rlen );
- hop.setDim2( clen );
- hop.setRowsInBlock( brlen );
- hop.setColsInBlock( bclen );
- hop.setNnz( nnz );
- }
-
- public static void setOutputParametersForScalar( Hop hop )
- {
- hop.setDim1( 0 );
- hop.setDim2( 0 );
- hop.setRowsInBlock( -1 );
- hop.setColsInBlock( -1 );
- hop.setNnz( -1 );
- }
-
- public static void refreshOutputParameters( Hop hnew, Hop hold )
- {
- hnew.setDim1( hold.getDim1() );
- hnew.setDim2( hold.getDim2() );
- hnew.setRowsInBlock(hold.getRowsInBlock());
- hnew.setColsInBlock(hold.getColsInBlock());
- hnew.refreshSizeInformation();
- }
-
- public static void copyLineNumbers( Hop src, Hop dest )
- {
- dest.setAllPositions(src.getBeginLine(), src.getBeginColumn(), src.getEndLine(), src.getEndColumn());
- }
-
- /**
- *
- * @param hop
- * @param brlen
- * @param bclen
- * @param src
- */
- public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, Hop src )
- {
- updateHopCharacteristics(hop, brlen, bclen, new MemoTable(), src);
- }
-
- /**
- *
- * @param hop
- * @param brlen
- * @param bclen
- * @param memo
- * @param src
- */
- public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, MemoTable memo, Hop src )
- {
- //update block sizes and dimensions
- setOutputBlocksizes(hop, brlen, bclen);
- hop.refreshSizeInformation();
-
- //compute memory estimates (for exec type selection)
- hop.computeMemEstimate(memo);
-
- //update line numbers
- HopRewriteUtils.copyLineNumbers(src, hop);
- }
-
- ///////////////////////////////////
- // hop size information
-
- public static boolean isDimsKnown( Hop hop )
- {
- return ( hop.getDim1()>0 && hop.getDim2()>0 );
- }
-
- public static boolean isEmpty( Hop hop )
- {
- return ( hop.getNnz()==0 );
- }
-
- public static boolean isEqualSize( Hop hop1, Hop hop2 )
- {
- return (hop1.dimsKnown() && hop2.dimsKnown()
- && hop1.getDim1() == hop2.getDim1()
- && hop1.getDim2() == hop2.getDim2());
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean isSingleBlock( Hop hop ) {
- return isSingleBlock(hop, true)
- && isSingleBlock(hop, false);
- }
-
-
- /**
- * Checks our BLOCKSIZE CONSTRAINT, w/ awareness of forced single node
- * execution mode.
- *
- * @param hop
- * @param cols
- * @return
- */
- public static boolean isSingleBlock( Hop hop, boolean cols )
- {
- //awareness of forced exec single node (e.g., standalone), where we can
- //guarantee a single block independent of the size because always in CP.
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE ) {
- return true;
- }
-
- //check row- or column-wise single block constraint
- return cols ? (hop.getDim2()>0 && hop.getDim2()<=hop.getColsInBlock())
- : (hop.getDim1()>0 && hop.getDim1()<=hop.getRowsInBlock());
- }
-
- public static boolean isEqualValue( LiteralOp hop1, LiteralOp hop2 )
- throws HopsException
- {
- //check for string (no defined double value)
- if( hop1.getValueType()==ValueType.STRING
- || hop2.getValueType()==ValueType.STRING )
- {
- return false;
- }
-
- double val1 = getDoubleValue(hop1);
- double val2 = getDoubleValue(hop2);
-
- return ( val1 == val2 );
- }
-
- public static boolean isNotMatrixVectorBinaryOperation( Hop hop )
- {
- boolean ret = true;
-
- if( hop instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp) hop;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
- boolean mv = (left.getDim1()>1 && right.getDim1()==1)
- || (left.getDim2()>1 && right.getDim2()==1);
- ret = isDimsKnown(bop) && !mv;
- }
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean isTransposeOperation(Hop hop)
- {
- return (hop instanceof ReorgOp && ((ReorgOp)hop).getOp()==ReOrgOp.TRANSPOSE);
- }
-
- /**
- *
- * @param hop1
- * @param hop2
- * @return
- */
- public static boolean isTransposeOfItself(Hop hop1, Hop hop2)
- {
- return hop1 instanceof ReorgOp && ((ReorgOp)hop1).getOp()==ReOrgOp.TRANSPOSE && hop1.getInput().get(0) == hop2
- || hop2 instanceof ReorgOp && ((ReorgOp)hop2).getOp()==ReOrgOp.TRANSPOSE && hop2.getInput().get(0) == hop1;
- }
-
- /**
- *
- * @param pred
- * @param hop
- * @return
- */
- public static boolean isNonZeroIndicator(Hop pred, Hop hop )
- {
- if( pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL
- && pred.getInput().get(0) == hop //depend on common subexpression elimination
- && pred.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred.getInput().get(1))==0 )
- {
- return true;
- }
-
- return false;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean isFullColumnIndexing(LeftIndexingOp hop)
- {
- boolean colPred = hop.getColLowerEqualsUpper(); //single col
-
- Hop rl = hop.getInput().get(2);
- Hop ru = hop.getInput().get(3);
-
- return colPred && rl instanceof LiteralOp && getDoubleValueSafe((LiteralOp)rl)==1
- && ru instanceof LiteralOp && getDoubleValueSafe((LiteralOp)ru)==hop.getDim1();
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean isFullRowIndexing(LeftIndexingOp hop)
- {
- boolean rowPred = hop.getRowLowerEqualsUpper(); //single row
-
- Hop cl = hop.getInput().get(4);
- Hop cu = hop.getInput().get(5);
-
- return rowPred && cl instanceof LiteralOp && getDoubleValueSafe((LiteralOp)cl)==1
- && cu instanceof LiteralOp && getDoubleValueSafe((LiteralOp)cu)==hop.getDim2();
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean isBasic1NSequence(Hop hop)
- {
- boolean ret = false;
-
- if( hop instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) hop;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop from = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_FROM));
- Hop incr = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_INCR));
- ret = (from instanceof LiteralOp && getDoubleValueSafe((LiteralOp)from)==1)
- &&(incr instanceof LiteralOp && getDoubleValueSafe((LiteralOp)incr)==1);
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @return
- * @throws HopsException
- */
- public static double getBasic1NSequenceMax(Hop hop)
- throws HopsException
- {
- if( hop instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) hop;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop to = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_TO));
- if( to instanceof LiteralOp )
- return getDoubleValueSafe((LiteralOp)to);
- }
- }
-
- throw new HopsException("Failed to retrieve 'to' argument from basic 1-N sequence.");
- }
-
- /**
- *
- * @param hop
- * @return
- * @throws HopsException
- */
- public static LiteralOp getBasic1NSequenceMaxLiteral(Hop hop)
- throws HopsException
- {
- if( hop instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) hop;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop to = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_TO));
- if( to instanceof LiteralOp )
- return (LiteralOp)to;
- }
- }
-
- throw new HopsException("Failed to retrieve 'to' argument from basic 1-N sequence.");
- }
-
-
- /**
- *
- * @param hop
- * @param inclTransient
- * @param inclPersistent
- * @return
- */
- public static boolean hasOnlyWriteParents( Hop hop, boolean inclTransient, boolean inclPersistent )
- {
- boolean ret = true;
-
- ArrayList<Hop> parents = hop.getParent();
- for( Hop p : parents )
- {
- if( inclTransient && inclPersistent )
- ret &= ( p instanceof DataOp && (((DataOp)p).getDataOpType()==DataOpTypes.TRANSIENTWRITE
- || ((DataOp)p).getDataOpType()==DataOpTypes.PERSISTENTWRITE));
- else if(inclTransient)
- ret &= ( p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.TRANSIENTWRITE);
- else if(inclPersistent)
- ret &= ( p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.PERSISTENTWRITE);
- }
-
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean hasTransformParents( Hop hop )
- {
- boolean ret = false;
-
- ArrayList<Hop> parents = hop.getParent();
- for( Hop p : parents )
- {
- if( p instanceof ParameterizedBuiltinOp
- && ((ParameterizedBuiltinOp)p).getOp()==ParamBuiltinOp.TRANSFORM) {
- ret = true;
- }
- }
-
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- public static boolean alwaysRequiresReblock(Hop hop)
- {
- return ( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType()==DataOpTypes.PERSISTENTREAD
- && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.BINARY);
- }
-
- /**
- *
- * @param root
- * @param var
- * @return
- */
- public static boolean rHasSimpleReadChain(Hop root, String var)
- {
- if( root.getVisited()==VisitStatus.DONE )
- return false;
-
- boolean ret = false;
-
- //handle leaf node for variable
- if( root instanceof DataOp && ((DataOp)root).isRead()
- && root.getName().equals(var) )
- {
- ret = (root.getParent().size()<=1);
- }
-
- //recursively process childs (on the entire path to var, all
- //intermediates are supposed to have at most one consumer, but
- //side-ways inputs can have arbitrary dag structures)
- for( Hop c : root.getInput() ) {
- if( rHasSimpleReadChain(c, var) )
- ret |= root.getParent().size()<=1;
- }
-
- root.setVisited(Hop.VisitStatus.DONE);
- return ret;
- }
-
- /**
- *
- * @param root
- * @param var
- * @param includeMetaOp
- * @return
- */
- public static boolean rContainsRead(Hop root, String var, boolean includeMetaOp)
- {
- if( root.getVisited()==VisitStatus.DONE )
- return false;
-
- boolean ret = false;
-
- //handle leaf node for variable
- if( root instanceof DataOp && ((DataOp)root).isRead()
- && root.getName().equals(var) )
- {
- boolean onlyMetaOp = true;
- if( !includeMetaOp ){
- for( Hop p : root.getParent() ) {
- onlyMetaOp &= (p instanceof UnaryOp
- && (((UnaryOp)p).getOp()==OpOp1.NROW
- || ((UnaryOp)p).getOp()==OpOp1.NCOL) );
- }
- ret = !onlyMetaOp;
- }
- else
- ret = true;
- }
-
- //recursively process childs
- for( Hop c : root.getInput() )
- ret |= rContainsRead(c, var, includeMetaOp);
-
- root.setVisited(Hop.VisitStatus.DONE);
- return ret;
- }
-
- //////////////////////////////////////
- // utils for lookup tables
-
- public static boolean isValidOp( AggOp input, AggOp[] validTab )
- {
- for( AggOp valid : validTab )
- if( valid == input )
- return true;
- return false;
- }
-
- public static boolean isValidOp( OpOp1 input, OpOp1[] validTab )
- {
- for( OpOp1 valid : validTab )
- if( valid == input )
- return true;
- return false;
- }
-
- public static boolean isValidOp( OpOp2 input, OpOp2[] validTab )
- {
- for( OpOp2 valid : validTab )
- if( valid == input )
- return true;
- return false;
- }
-
- public static int getValidOpPos( OpOp2 input, OpOp2[] validTab )
- {
- for( int i=0; i<validTab.length; i++ ) {
- OpOp2 valid = validTab[i];
- if( valid == input )
- return i;
- }
- return -1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriteStatus.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriteStatus.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriteStatus.java
deleted file mode 100644
index 535f68c..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriteStatus.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-public class ProgramRewriteStatus
-{
-
- //status of applied rewrites
- private boolean _rmBranches = false; //removed branches
- private int _blkSize = -1;
- private boolean _injectCheckpoints = false;
-
- //current context
- private boolean _inParforCtx = false;
-
- public ProgramRewriteStatus()
- {
- _rmBranches = false;
- _inParforCtx = false;
- _injectCheckpoints = false;
- }
-
- public void setRemovedBranches(){
- _rmBranches = true;
- }
-
- public boolean getRemovedBranches(){
- return _rmBranches;
- }
-
- public void setInParforContext(boolean flag){
- _inParforCtx = flag;
- }
-
- public boolean isInParforContext(){
- return _inParforCtx;
- }
-
- public void setBlocksize( int blkSize ){
- _blkSize = blkSize;
- }
-
- public int getBlocksize() {
- return _blkSize;
- }
-
- public void setInjectedCheckpoints(){
- _injectCheckpoints = true;
- }
-
- public boolean getInjectedCheckpoints(){
- return _injectCheckpoints;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriter.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriter.java
deleted file mode 100644
index 9c383e9..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/ProgramRewriter.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-
-/**
- * This program rewriter applies a variety of rule-based rewrites
- * on all hop dags of the given program in one pass over the entire
- * program.
- *
- */
-public class ProgramRewriter
-{
- private static final Log LOG = LogFactory.getLog(ProgramRewriter.class.getName());
-
- //internal local debug level
- private static final boolean LDEBUG = false;
- private static final boolean CHECK = false;
-
- private ArrayList<HopRewriteRule> _dagRuleSet = null;
- private ArrayList<StatementBlockRewriteRule> _sbRuleSet = null;
-
- static{
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.rewrite")
- .setLevel((Level) Level.DEBUG);
- }
-
- }
-
- public ProgramRewriter()
- {
- // by default which is used during initial compile
- // apply all (static and dynamic) rewrites
- this( true, true );
- }
-
- public ProgramRewriter( boolean staticRewrites, boolean dynamicRewrites )
- {
- //initialize HOP DAG rewrite ruleSet (with fixed rewrite order)
- _dagRuleSet = new ArrayList<HopRewriteRule>();
-
- //initialize StatementBlock rewrite ruleSet (with fixed rewrite order)
- _sbRuleSet = new ArrayList<StatementBlockRewriteRule>();
-
-
- //STATIC REWRITES (which do not rely on size information)
- if( staticRewrites )
- {
- //add static HOP DAG rewrite rules
- _dagRuleSet.add( new RewriteTransientWriteParentHandling() );
- _dagRuleSet.add( new RewriteRemoveReadAfterWrite() ); //dependency: before blocksize
- _dagRuleSet.add( new RewriteBlockSizeAndReblock() );
- _dagRuleSet.add( new RewriteRemoveUnnecessaryCasts() );
- if( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
- _dagRuleSet.add( new RewriteCommonSubexpressionElimination() );
- if( OptimizerUtils.ALLOW_CONSTANT_FOLDING )
- _dagRuleSet.add( new RewriteConstantFolding() ); //dependency: cse
- if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )
- _dagRuleSet.add( new RewriteAlgebraicSimplificationStatic() ); //dependencies: cse
- if( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION ) //dependency: simplifications (no need to merge leafs again)
- _dagRuleSet.add( new RewriteCommonSubexpressionElimination() );
- if( OptimizerUtils.ALLOW_AUTO_VECTORIZATION )
- _dagRuleSet.add( new RewriteIndexingVectorization() ); //dependency: cse, simplifications
- _dagRuleSet.add( new RewriteInjectSparkPReadCheckpointing() ); //dependency: reblock
-
- //add statment block rewrite rules
- if( OptimizerUtils.ALLOW_BRANCH_REMOVAL )
- _sbRuleSet.add( new RewriteRemoveUnnecessaryBranches() ); //dependency: constant folding
- if( OptimizerUtils.ALLOW_SPLIT_HOP_DAGS )
- _sbRuleSet.add( new RewriteSplitDagUnknownCSVRead() ); //dependency: reblock
- if( OptimizerUtils.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS )
- _sbRuleSet.add( new RewriteSplitDagDataDependentOperators() );
- if( OptimizerUtils.ALLOW_AUTO_VECTORIZATION )
- _sbRuleSet.add( new RewriteForLoopVectorization() ); //dependency: reblock (reblockop)
- _sbRuleSet.add( new RewriteInjectSparkLoopCheckpointing(true) ); //dependency: reblock (blocksizes)
- }
-
- // DYNAMIC REWRITES (which do require size information)
- if( dynamicRewrites )
- {
- _dagRuleSet.add( new RewriteMatrixMultChainOptimization() ); //dependency: cse
-
- if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )
- {
- _dagRuleSet.add( new RewriteAlgebraicSimplificationDynamic() ); //dependencies: cse
- _dagRuleSet.add( new RewriteAlgebraicSimplificationStatic() ); //dependencies: cse
- }
-
- //reapply cse after rewrites because (1) applied rewrites on operators w/ multiple parents, and
- //(2) newly introduced operators potentially created redundancy (incl leaf merge to allow for cse)
- if( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
- _dagRuleSet.add( new RewriteCommonSubexpressionElimination(true) ); //dependency: simplifications
- }
- }
-
- /**
- * Construct a program rewriter for a given rewrite which is passed from outside.
- *
- * @param rewrite
- */
- public ProgramRewriter( HopRewriteRule rewrite )
- {
- //initialize HOP DAG rewrite ruleSet (with fixed rewrite order)
- _dagRuleSet = new ArrayList<HopRewriteRule>();
- _dagRuleSet.add( rewrite );
-
- _sbRuleSet = new ArrayList<StatementBlockRewriteRule>();
- }
-
- /**
- * Construct a program rewriter for a given rewrite which is passed from outside.
- *
- * @param rewrite
- */
- public ProgramRewriter( StatementBlockRewriteRule rewrite )
- {
- //initialize HOP DAG rewrite ruleSet (with fixed rewrite order)
- _dagRuleSet = new ArrayList<HopRewriteRule>();
-
- _sbRuleSet = new ArrayList<StatementBlockRewriteRule>();
- _sbRuleSet.add( rewrite );
- }
-
- /**
- * Construct a program rewriter for the given rewrite sets which are passed from outside.
- *
- * @param rewrite
- */
- public ProgramRewriter( ArrayList<HopRewriteRule> hRewrites, ArrayList<StatementBlockRewriteRule> sbRewrites )
- {
- //initialize HOP DAG rewrite ruleSet (with fixed rewrite order)
- _dagRuleSet = new ArrayList<HopRewriteRule>();
- _dagRuleSet.addAll( hRewrites );
-
- _sbRuleSet = new ArrayList<StatementBlockRewriteRule>();
- _sbRuleSet.addAll( sbRewrites );
- }
-
- /**
- *
- * @param dmlp
- * @return
- * @throws LanguageException
- * @throws HopsException
- */
- public ProgramRewriteStatus rewriteProgramHopDAGs(DMLProgram dmlp)
- throws LanguageException, HopsException
- {
- ProgramRewriteStatus state = new ProgramRewriteStatus();
-
- // for each namespace, handle function statement blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet())
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet())
- {
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- rewriteStatementBlockHopDAGs(fsblock, state);
- rewriteStatementBlock(fsblock, state);
- }
-
- // handle regular statement blocks in "main" method
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++)
- {
- StatementBlock current = dmlp.getStatementBlock(i);
- rewriteStatementBlockHopDAGs(current, state);
- }
- dmlp.setStatementBlocks( rewriteStatementBlocks(dmlp.getStatementBlocks(), state) );
-
- return state;
- }
-
- /**
- *
- * @param current
- * @throws LanguageException
- * @throws HopsException
- */
- public void rewriteStatementBlockHopDAGs(StatementBlock current, ProgramRewriteStatus state)
- throws LanguageException, HopsException
- {
- //ensure robustness for calls from outside
- if( state == null )
- state = new ProgramRewriteStatus();
-
- if (current instanceof FunctionStatementBlock)
- {
- FunctionStatementBlock fsb = (FunctionStatementBlock)current;
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- for (StatementBlock sb : fstmt.getBody())
- rewriteStatementBlockHopDAGs(sb, state);
- }
- else if (current instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock) current;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- wsb.setPredicateHops(rewriteHopDAG(wsb.getPredicateHops(), state));
- for (StatementBlock sb : wstmt.getBody())
- rewriteStatementBlockHopDAGs(sb, state);
- }
- else if (current instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) current;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- isb.setPredicateHops(rewriteHopDAG(isb.getPredicateHops(), state));
- for (StatementBlock sb : istmt.getIfBody())
- rewriteStatementBlockHopDAGs(sb, state);
- for (StatementBlock sb : istmt.getElseBody())
- rewriteStatementBlockHopDAGs(sb, state);
- }
- else if (current instanceof ForStatementBlock) //incl parfor
- {
- ForStatementBlock fsb = (ForStatementBlock) current;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- fsb.setFromHops(rewriteHopDAG(fsb.getFromHops(), state));
- fsb.setToHops(rewriteHopDAG(fsb.getToHops(), state));
- fsb.setIncrementHops(rewriteHopDAG(fsb.getIncrementHops(), state));
- for (StatementBlock sb : fstmt.getBody())
- rewriteStatementBlockHopDAGs(sb, state);
- }
- else //generic (last-level)
- {
- current.set_hops( rewriteHopDAGs(current.get_hops(), state) );
- }
- }
-
- /**
- *
- * @param roots
- * @throws LanguageException
- * @throws HopsException
- */
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- for( HopRewriteRule r : _dagRuleSet )
- {
- Hop.resetVisitStatus( roots ); //reset for each rule
- roots = r.rewriteHopDAGs(roots, state);
-
- if( CHECK ) {
- LOG.info("Validation after: "+r.getClass().getName());
- HopDagValidator.validateHopDag(roots);
- }
- }
-
- return roots;
- }
-
- /**
- *
- * @param root
- * @throws LanguageException
- * @throws HopsException
- */
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- for( HopRewriteRule r : _dagRuleSet )
- {
- root.resetVisitStatus(); //reset for each rule
- root = r.rewriteHopDAG(root, state);
-
- if( CHECK ) {
- LOG.info("Validation after: "+r.getClass().getName());
- HopDagValidator.validateHopDag(root);
- }
- }
-
- return root;
- }
-
- /**
- *
- * @param sbs
- * @return
- * @throws HopsException
- */
- public ArrayList<StatementBlock> rewriteStatementBlocks( ArrayList<StatementBlock> sbs, ProgramRewriteStatus state )
- throws HopsException
- {
- //ensure robustness for calls from outside
- if( state == null )
- state = new ProgramRewriteStatus();
-
-
- ArrayList<StatementBlock> tmp = new ArrayList<StatementBlock>();
-
- //rewrite statement blocks (with potential expansion)
- for( StatementBlock sb : sbs )
- tmp.addAll( rewriteStatementBlock(sb, state) );
-
- //copy results into original collection
- sbs.clear();
- sbs.addAll( tmp );
-
- return sbs;
- }
-
- /**
- *
- * @param sb
- * @return
- * @throws HopsException
- */
- private ArrayList<StatementBlock> rewriteStatementBlock( StatementBlock sb, ProgramRewriteStatus status )
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
- ret.add(sb);
-
- //recursive invocation
- if (sb instanceof FunctionStatementBlock)
- {
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- fstmt.setBody( rewriteStatementBlocks(fstmt.getBody(), status) );
- }
- else if (sb instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- wstmt.setBody( rewriteStatementBlocks( wstmt.getBody(), status ) );
- }
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- istmt.setIfBody( rewriteStatementBlocks( istmt.getIfBody(), status ) );
- istmt.setElseBody( rewriteStatementBlocks( istmt.getElseBody(), status ) );
- }
- else if (sb instanceof ForStatementBlock) //incl parfor
- {
- //maintain parfor context information (e.g., for checkpointing)
- boolean prestatus = status.isInParforContext();
- if( sb instanceof ParForStatementBlock )
- status.setInParforContext(true);
-
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- fstmt.setBody( rewriteStatementBlocks(fstmt.getBody(), status) );
-
- status.setInParforContext(prestatus);
- }
-
- //apply rewrite rules
- for( StatementBlockRewriteRule r : _sbRuleSet )
- {
- ArrayList<StatementBlock> tmp = new ArrayList<StatementBlock>();
- for( StatementBlock sbc : ret )
- tmp.addAll( r.rewriteStatementBlock(sbc, status) );
-
- //take over set of rewritten sbs
- ret.clear();
- ret.addAll(tmp);
- }
-
- return ret;
- }
-}
[51/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
Move files to new package folder structure
The refactor from com.ibm.bi.dml to org.apache.sysml
is being done in two phases to avoid loosing the file
history. In this phase, files are being moved to new
location.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/276d9257
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/276d9257
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/276d9257
Branch: refs/heads/master
Commit: 276d9257c08e667bc70ce49024c6450deb473b43
Parents: 7e295c1
Author: Luciano Resende <lr...@apache.org>
Authored: Tue Dec 1 13:39:01 2015 -0800
Committer: Luciano Resende <lr...@apache.org>
Committed: Thu Dec 3 10:43:48 2015 -0800
----------------------------------------------------------------------
.../java/com/ibm/bi/dml/api/DMLException.java | 58 -
src/main/java/com/ibm/bi/dml/api/DMLScript.java | 990 ---
src/main/java/com/ibm/bi/dml/api/MLBlock.java | 241 -
src/main/java/com/ibm/bi/dml/api/MLContext.java | 1396 ----
.../java/com/ibm/bi/dml/api/MLContextProxy.java | 102 -
src/main/java/com/ibm/bi/dml/api/MLMatrix.java | 408 --
src/main/java/com/ibm/bi/dml/api/MLOutput.java | 401 --
.../com/ibm/bi/dml/api/jmlc/Connection.java | 244 -
.../java/com/ibm/bi/dml/api/jmlc/JMLCUtils.java | 107 -
.../com/ibm/bi/dml/api/jmlc/PreparedScript.java | 231 -
.../ibm/bi/dml/api/jmlc/ResultVariables.java | 92 -
.../ibm/bi/dml/api/ml/LogisticRegression.java | 464 --
.../bi/dml/api/ml/LogisticRegressionModel.java | 169 -
.../functions/ConvertSingleColumnToString.java | 15 -
.../api/ml/functions/ConvertVectorToDouble.java | 16 -
.../api/monitoring/InstructionComparator.java | 38 -
.../com/ibm/bi/dml/api/monitoring/Location.java | 77 -
.../dml/api/monitoring/SparkMonitoringUtil.java | 602 --
.../java/com/ibm/bi/dml/api/python/SystemML.py | 198 -
.../ibm/bi/dml/conf/ConfigurationManager.java | 79 -
.../java/com/ibm/bi/dml/conf/DMLConfig.java | 491 --
.../ibm/bi/dml/debug/DMLBreakpointManager.java | 191 -
.../java/com/ibm/bi/dml/debug/DMLDebugger.java | 609 --
.../ibm/bi/dml/debug/DMLDebuggerException.java | 41 -
.../ibm/bi/dml/debug/DMLDebuggerFunctions.java | 678 --
.../ibm/bi/dml/debug/DMLDebuggerInterface.java | 316 -
.../bi/dml/debug/DMLDebuggerProgramInfo.java | 375 --
.../com/ibm/bi/dml/debug/DMLDisassemble.java | 201 -
.../java/com/ibm/bi/dml/debug/DMLFrame.java | 68 -
.../com/ibm/bi/dml/debug/DMLProgramCounter.java | 208 -
.../java/com/ibm/bi/dml/debug/DebugState.java | 190 -
.../java/com/ibm/bi/dml/hops/AggBinaryOp.java | 1959 ------
.../java/com/ibm/bi/dml/hops/AggUnaryOp.java | 735 --
src/main/java/com/ibm/bi/dml/hops/BinaryOp.java | 1565 -----
.../java/com/ibm/bi/dml/hops/DataGenOp.java | 532 --
src/main/java/com/ibm/bi/dml/hops/DataOp.java | 529 --
.../java/com/ibm/bi/dml/hops/FunctionOp.java | 297 -
src/main/java/com/ibm/bi/dml/hops/Hop.java | 1882 ------
.../java/com/ibm/bi/dml/hops/HopsException.java | 58 -
.../java/com/ibm/bi/dml/hops/IndexingOp.java | 464 --
.../com/ibm/bi/dml/hops/LeftIndexingOp.java | 503 --
.../java/com/ibm/bi/dml/hops/LiteralOp.java | 291 -
.../java/com/ibm/bi/dml/hops/MemoTable.java | 243 -
.../com/ibm/bi/dml/hops/OptimizerUtils.java | 1407 ----
.../ibm/bi/dml/hops/ParameterizedBuiltinOp.java | 1139 ----
.../java/com/ibm/bi/dml/hops/QuaternaryOp.java | 1684 -----
src/main/java/com/ibm/bi/dml/hops/ReorgOp.java | 608 --
.../java/com/ibm/bi/dml/hops/TernaryOp.java | 1024 ---
src/main/java/com/ibm/bi/dml/hops/UnaryOp.java | 746 ---
.../bi/dml/hops/cost/CostEstimationWrapper.java | 161 -
.../com/ibm/bi/dml/hops/cost/CostEstimator.java | 787 ---
.../dml/hops/cost/CostEstimatorNumMRJobs.java | 44 -
.../hops/cost/CostEstimatorStaticRuntime.java | 1317 ----
.../java/com/ibm/bi/dml/hops/cost/VarStats.java | 77 -
.../bi/dml/hops/globalopt/GDFEnumOptimizer.java | 603 --
.../bi/dml/hops/globalopt/GlobalOptimizer.java | 47 -
.../hops/globalopt/GlobalOptimizerWrapper.java | 135 -
.../hops/globalopt/InterestingProperties.java | 118 -
.../bi/dml/hops/globalopt/MemoStructure.java | 94 -
.../com/ibm/bi/dml/hops/globalopt/Plan.java | 228 -
.../com/ibm/bi/dml/hops/globalopt/PlanSet.java | 168 -
.../bi/dml/hops/globalopt/RewriteConfig.java | 96 -
.../com/ibm/bi/dml/hops/globalopt/Summary.java | 99 -
.../globalopt/gdfgraph/GDFCrossBlockNode.java | 93 -
.../dml/hops/globalopt/gdfgraph/GDFGraph.java | 47 -
.../hops/globalopt/gdfgraph/GDFLoopNode.java | 64 -
.../bi/dml/hops/globalopt/gdfgraph/GDFNode.java | 174 -
.../hops/globalopt/gdfgraph/GraphBuilder.java | 350 -
.../gdfresolve/GDFMismatchHeuristic.java | 46 -
.../GDFMismatchHeuristicBlocksizeOrFirst.java | 47 -
.../gdfresolve/GDFMismatchHeuristicFirst.java | 36 -
.../gdfresolve/MismatchHeuristicFactory.java | 45 -
.../dml/hops/ipa/InterProceduralAnalysis.java | 1248 ----
.../dml/hops/recompile/LiteralReplacement.java | 538 --
.../bi/dml/hops/recompile/RecompileStatus.java | 61 -
.../ibm/bi/dml/hops/recompile/Recompiler.java | 2126 ------
.../bi/dml/hops/rewrite/HopDagValidator.java | 126 -
.../ibm/bi/dml/hops/rewrite/HopRewriteRule.java | 60 -
.../bi/dml/hops/rewrite/HopRewriteUtils.java | 1071 ---
.../dml/hops/rewrite/ProgramRewriteStatus.java | 69 -
.../bi/dml/hops/rewrite/ProgramRewriter.java | 397 --
.../RewriteAlgebraicSimplificationDynamic.java | 2506 -------
.../RewriteAlgebraicSimplificationStatic.java | 1544 -----
.../rewrite/RewriteBlockSizeAndReblock.java | 242 -
.../RewriteCommonSubexpressionElimination.java | 232 -
.../hops/rewrite/RewriteConstantFolding.java | 333 -
.../rewrite/RewriteForLoopVectorization.java | 398 --
.../rewrite/RewriteIndexingVectorization.java | 359 -
.../RewriteInjectSparkLoopCheckpointing.java | 121 -
.../RewriteInjectSparkPReadCheckpointing.java | 94 -
.../RewriteMatrixMultChainOptimization.java | 423 --
.../RewriteRemovePersistentReadWrite.java | 117 -
.../rewrite/RewriteRemoveReadAfterWrite.java | 124 -
.../RewriteRemoveUnnecessaryBranches.java | 82 -
.../rewrite/RewriteRemoveUnnecessaryCasts.java | 120 -
.../RewriteSplitDagDataDependentOperators.java | 519 --
.../rewrite/RewriteSplitDagUnknownCSVRead.java | 172 -
.../RewriteTransientWriteParentHandling.java | 89 -
.../hops/rewrite/StatementBlockRewriteRule.java | 50 -
.../java/com/ibm/bi/dml/lops/Aggregate.java | 190 -
src/main/java/com/ibm/bi/dml/lops/AppendCP.java | 91 -
src/main/java/com/ibm/bi/dml/lops/AppendG.java | 128 -
.../com/ibm/bi/dml/lops/AppendGAlignedSP.java | 74 -
src/main/java/com/ibm/bi/dml/lops/AppendM.java | 132 -
src/main/java/com/ibm/bi/dml/lops/AppendR.java | 104 -
src/main/java/com/ibm/bi/dml/lops/Binary.java | 194 -
src/main/java/com/ibm/bi/dml/lops/BinaryM.java | 218 -
.../java/com/ibm/bi/dml/lops/BinaryScalar.java | 199 -
.../com/ibm/bi/dml/lops/BinaryUAggChain.java | 145 -
.../java/com/ibm/bi/dml/lops/CSVReBlock.java | 193 -
.../java/com/ibm/bi/dml/lops/CentralMoment.java | 162 -
.../java/com/ibm/bi/dml/lops/Checkpoint.java | 167 -
.../java/com/ibm/bi/dml/lops/CoVariance.java | 179 -
.../java/com/ibm/bi/dml/lops/CombineBinary.java | 127 -
.../com/ibm/bi/dml/lops/CombineTernary.java | 120 -
.../java/com/ibm/bi/dml/lops/CombineUnary.java | 99 -
.../ibm/bi/dml/lops/CumulativeOffsetBinary.java | 165 -
.../bi/dml/lops/CumulativePartialAggregate.java | 137 -
.../bi/dml/lops/CumulativeSplitAggregate.java | 87 -
src/main/java/com/ibm/bi/dml/lops/Data.java | 619 --
src/main/java/com/ibm/bi/dml/lops/DataGen.java | 558 --
.../java/com/ibm/bi/dml/lops/DataPartition.java | 115 -
.../com/ibm/bi/dml/lops/FunctionCallCP.java | 150 -
src/main/java/com/ibm/bi/dml/lops/Group.java | 73 -
.../com/ibm/bi/dml/lops/GroupedAggregate.java | 244 -
.../java/com/ibm/bi/dml/lops/LeftIndex.java | 178 -
src/main/java/com/ibm/bi/dml/lops/Lop.java | 720 --
.../java/com/ibm/bi/dml/lops/LopProperties.java | 172 -
.../java/com/ibm/bi/dml/lops/LopsException.java | 40 -
src/main/java/com/ibm/bi/dml/lops/MMCJ.java | 148 -
src/main/java/com/ibm/bi/dml/lops/MMRJ.java | 106 -
src/main/java/com/ibm/bi/dml/lops/MMTSJ.java | 122 -
src/main/java/com/ibm/bi/dml/lops/MMZip.java | 83 -
src/main/java/com/ibm/bi/dml/lops/MapMult.java | 213 -
.../java/com/ibm/bi/dml/lops/MapMultChain.java | 268 -
.../com/ibm/bi/dml/lops/OutputParameters.java | 158 -
src/main/java/com/ibm/bi/dml/lops/PMMJ.java | 164 -
src/main/java/com/ibm/bi/dml/lops/PMapMult.java | 86 -
.../ibm/bi/dml/lops/ParameterizedBuiltin.java | 514 --
.../com/ibm/bi/dml/lops/PartialAggregate.java | 414 --
.../java/com/ibm/bi/dml/lops/PickByCount.java | 182 -
.../com/ibm/bi/dml/lops/RangeBasedReIndex.java | 198 -
src/main/java/com/ibm/bi/dml/lops/ReBlock.java | 168 -
src/main/java/com/ibm/bi/dml/lops/RepMat.java | 91 -
src/main/java/com/ibm/bi/dml/lops/SortKeys.java | 183 -
src/main/java/com/ibm/bi/dml/lops/Ternary.java | 385 --
.../com/ibm/bi/dml/lops/TernaryAggregate.java | 104 -
.../java/com/ibm/bi/dml/lops/Transform.java | 286 -
.../com/ibm/bi/dml/lops/UAggOuterChain.java | 170 -
src/main/java/com/ibm/bi/dml/lops/Unary.java | 445 --
src/main/java/com/ibm/bi/dml/lops/UnaryCP.java | 178 -
.../ibm/bi/dml/lops/WeightedCrossEntropy.java | 156 -
.../ibm/bi/dml/lops/WeightedCrossEntropyR.java | 156 -
.../java/com/ibm/bi/dml/lops/WeightedDivMM.java | 186 -
.../com/ibm/bi/dml/lops/WeightedDivMMR.java | 155 -
.../com/ibm/bi/dml/lops/WeightedSigmoid.java | 160 -
.../com/ibm/bi/dml/lops/WeightedSigmoidR.java | 156 -
.../ibm/bi/dml/lops/WeightedSquaredLoss.java | 169 -
.../ibm/bi/dml/lops/WeightedSquaredLossR.java | 160 -
.../com/ibm/bi/dml/lops/WeightedUnaryMM.java | 165 -
.../com/ibm/bi/dml/lops/WeightedUnaryMMR.java | 162 -
src/main/java/com/ibm/bi/dml/lops/ZeroOut.java | 132 -
.../java/com/ibm/bi/dml/lops/compile/Dag.java | 4574 -------------
.../com/ibm/bi/dml/lops/compile/JobType.java | 252 -
.../ibm/bi/dml/lops/compile/LopComparator.java | 62 -
.../com/ibm/bi/dml/lops/runtime/RunMRJobs.java | 591 --
.../com/ibm/bi/dml/parser/AParserWrapper.java | 62 -
.../ibm/bi/dml/parser/AssignmentStatement.java | 146 -
.../com/ibm/bi/dml/parser/BinaryExpression.java | 227 -
.../ibm/bi/dml/parser/BooleanExpression.java | 159 -
.../ibm/bi/dml/parser/BooleanIdentifier.java | 64 -
.../dml/parser/BuiltinFunctionExpression.java | 1425 ----
.../ibm/bi/dml/parser/ConditionalPredicate.java | 92 -
.../com/ibm/bi/dml/parser/ConstIdentifier.java | 29 -
.../ibm/bi/dml/parser/DMLParseException.java | 87 -
.../java/com/ibm/bi/dml/parser/DMLProgram.java | 783 ---
.../com/ibm/bi/dml/parser/DMLTranslator.java | 2814 --------
.../com/ibm/bi/dml/parser/DataExpression.java | 2171 ------
.../com/ibm/bi/dml/parser/DataIdentifier.java | 174 -
.../com/ibm/bi/dml/parser/DoubleIdentifier.java | 85 -
.../java/com/ibm/bi/dml/parser/Expression.java | 513 --
.../dml/parser/ExternalFunctionStatement.java | 173 -
.../com/ibm/bi/dml/parser/ForStatement.java | 107 -
.../ibm/bi/dml/parser/ForStatementBlock.java | 444 --
.../bi/dml/parser/FunctionCallIdentifier.java | 208 -
.../ibm/bi/dml/parser/FunctionStatement.java | 154 -
.../bi/dml/parser/FunctionStatementBlock.java | 321 -
.../java/com/ibm/bi/dml/parser/Identifier.java | 281 -
.../java/com/ibm/bi/dml/parser/IfStatement.java | 162 -
.../com/ibm/bi/dml/parser/IfStatementBlock.java | 537 --
.../com/ibm/bi/dml/parser/ImportStatement.java | 113 -
.../ibm/bi/dml/parser/IndexedIdentifier.java | 782 ---
.../com/ibm/bi/dml/parser/IntIdentifier.java | 78 -
.../ibm/bi/dml/parser/IterablePredicate.java | 219 -
.../ibm/bi/dml/parser/LanguageException.java | 69 -
.../ibm/bi/dml/parser/LiveVariableAnalysis.java | 99 -
.../bi/dml/parser/MultiAssignmentStatement.java | 151 -
.../com/ibm/bi/dml/parser/OutputStatement.java | 233 -
.../com/ibm/bi/dml/parser/ParForStatement.java | 44 -
.../ibm/bi/dml/parser/ParForStatementBlock.java | 2177 ------
.../ibm/bi/dml/parser/ParameterExpression.java | 53 -
.../ParameterizedBuiltinFunctionExpression.java | 615 --
.../com/ibm/bi/dml/parser/ParseException.java | 87 -
.../com/ibm/bi/dml/parser/PathStatement.java | 70 -
.../com/ibm/bi/dml/parser/PrintStatement.java | 121 -
.../ibm/bi/dml/parser/RelationalExpression.java | 219 -
.../java/com/ibm/bi/dml/parser/Statement.java | 134 -
.../com/ibm/bi/dml/parser/StatementBlock.java | 1063 ---
.../com/ibm/bi/dml/parser/StringIdentifier.java | 75 -
src/main/java/com/ibm/bi/dml/parser/Token.java | 131 -
.../java/com/ibm/bi/dml/parser/VariableSet.java | 93 -
.../com/ibm/bi/dml/parser/WhileStatement.java | 106 -
.../ibm/bi/dml/parser/WhileStatementBlock.java | 334 -
.../bi/dml/parser/antlr4/DMLParserWrapper.java | 384 --
.../java/com/ibm/bi/dml/parser/antlr4/Dml.g4 | 199 -
.../antlr4/DmlSyntacticErrorListener.java | 113 -
.../parser/antlr4/DmlSyntacticValidator.java | 1517 -----
.../antlr4/DmlSyntacticValidatorHelper.java | 102 -
.../bi/dml/parser/antlr4/ExpressionInfo.java | 29 -
.../ibm/bi/dml/parser/antlr4/StatementInfo.java | 34 -
.../bi/dml/parser/python/ExpressionInfo.java | 13 -
.../dml/parser/python/PyDMLParserWrapper.java | 307 -
.../java/com/ibm/bi/dml/parser/python/Pydml.g4 | 382 --
.../python/PydmlSyntacticErrorListener.java | 113 -
.../parser/python/PydmlSyntacticValidator.java | 2107 ------
.../python/PydmlSyntacticValidatorHelper.java | 123 -
.../ibm/bi/dml/parser/python/StatementInfo.java | 13 -
.../ibm/bi/dml/runtime/DMLRuntimeException.java | 41 -
.../ibm/bi/dml/runtime/DMLScriptException.java | 46 -
.../DMLUnsupportedOperationException.java | 30 -
.../ExternalFunctionProgramBlock.java | 1064 ---
.../ExternalFunctionProgramBlockCP.java | 192 -
.../runtime/controlprogram/ForProgramBlock.java | 288 -
.../controlprogram/FunctionProgramBlock.java | 166 -
.../runtime/controlprogram/IfProgramBlock.java | 276 -
.../controlprogram/LocalVariableMap.java | 231 -
.../controlprogram/ParForProgramBlock.java | 2126 ------
.../bi/dml/runtime/controlprogram/Program.java | 164 -
.../runtime/controlprogram/ProgramBlock.java | 436 --
.../controlprogram/WhileProgramBlock.java | 233 -
.../controlprogram/caching/ByteBuffer.java | 210 -
.../controlprogram/caching/CacheDataInput.java | 232 -
.../controlprogram/caching/CacheDataOutput.java | 226 -
.../controlprogram/caching/CacheException.java | 48 -
.../caching/CacheIOException.java | 47 -
.../controlprogram/caching/CacheStatistics.java | 299 -
.../caching/CacheStatusException.java | 25 -
.../controlprogram/caching/CacheableData.java | 450 --
.../controlprogram/caching/LazyWriteBuffer.java | 278 -
.../controlprogram/caching/MatrixObject.java | 1686 -----
.../controlprogram/caching/PageCache.java | 86 -
.../context/ExecutionContext.java | 522 --
.../context/ExecutionContextFactory.java | 65 -
.../context/SparkExecutionContext.java | 1155 ----
.../controlprogram/parfor/DataPartitioner.java | 236 -
.../parfor/DataPartitionerLocal.java | 888 ---
.../parfor/DataPartitionerRemoteMR.java | 214 -
.../parfor/DataPartitionerRemoteMapper.java | 707 --
.../parfor/DataPartitionerRemoteReducer.java | 231 -
.../parfor/DataPartitionerRemoteSpark.java | 125 -
.../DataPartitionerRemoteSparkMapper.java | 153 -
.../DataPartitionerRemoteSparkReducer.java | 87 -
.../controlprogram/parfor/LocalParWorker.java | 148 -
.../controlprogram/parfor/LocalTaskQueue.java | 144 -
.../controlprogram/parfor/ParForBody.java | 85 -
.../controlprogram/parfor/ParWorker.java | 258 -
.../controlprogram/parfor/ProgramConverter.java | 2368 -------
.../controlprogram/parfor/RemoteDPParForMR.java | 290 -
.../parfor/RemoteDPParForSpark.java | 119 -
.../parfor/RemoteDPParForSparkWorker.java | 316 -
.../parfor/RemoteDPParWorkerReducer.java | 370 -
.../parfor/RemoteParForColocatedFileSplit.java | 183 -
.../RemoteParForColocatedNLineInputFormat.java | 69 -
.../parfor/RemoteParForJobReturn.java | 87 -
.../controlprogram/parfor/RemoteParForMR.java | 311 -
.../parfor/RemoteParForSpark.java | 106 -
.../parfor/RemoteParForSparkWorker.java | 131 -
.../parfor/RemoteParForUtils.java | 264 -
.../parfor/RemoteParWorkerMapper.java | 258 -
.../controlprogram/parfor/ResultMerge.java | 176 -
.../parfor/ResultMergeLocalAutomatic.java | 78 -
.../parfor/ResultMergeLocalFile.java | 1174 ----
.../parfor/ResultMergeLocalMemory.java | 338 -
.../parfor/ResultMergeRemoteGrouping.java | 43 -
.../parfor/ResultMergeRemoteMR.java | 339 -
.../parfor/ResultMergeRemoteMapper.java | 199 -
.../parfor/ResultMergeRemotePartitioning.java | 63 -
.../parfor/ResultMergeRemoteReducer.java | 359 -
.../parfor/ResultMergeRemoteSorting.java | 49 -
.../parfor/ResultMergeRemoteSpark.java | 219 -
.../parfor/ResultMergeRemoteSparkWCompare.java | 73 -
.../parfor/ResultMergeTaggedMatrixIndexes.java | 116 -
.../dml/runtime/controlprogram/parfor/Task.java | 235 -
.../controlprogram/parfor/TaskPartitioner.java | 84 -
.../parfor/TaskPartitionerFactoring.java | 208 -
.../parfor/TaskPartitionerFactoringCmax.java | 50 -
.../parfor/TaskPartitionerFactoringCmin.java | 49 -
.../parfor/TaskPartitionerFixedsize.java | 158 -
.../parfor/TaskPartitionerNaive.java | 38 -
.../parfor/TaskPartitionerStatic.java | 38 -
.../parfor/mqo/MergedMRJobInstruction.java | 76 -
.../parfor/mqo/PiggybackingWorker.java | 153 -
.../mqo/PiggybackingWorkerTimeSequential.java | 101 -
.../PiggybackingWorkerUtilDecayParallel.java | 182 -
.../mqo/PiggybackingWorkerUtilTimeParallel.java | 169 -
.../parfor/mqo/RuntimePiggybacking.java | 252 -
.../parfor/mqo/RuntimePiggybackingUtils.java | 52 -
.../parfor/opt/CostEstimator.java | 330 -
.../parfor/opt/CostEstimatorHops.java | 140 -
.../parfor/opt/CostEstimatorRuntime.java | 275 -
.../controlprogram/parfor/opt/CostFunction.java | 172 -
.../controlprogram/parfor/opt/MemoTable.java | 160 -
.../parfor/opt/MemoTableEntry.java | 112 -
.../controlprogram/parfor/opt/OptNode.java | 719 --
.../parfor/opt/OptNodeStatistics.java | 128 -
.../controlprogram/parfor/opt/OptTree.java | 111 -
.../parfor/opt/OptTreeConverter.java | 1104 ---
.../parfor/opt/OptTreePlanChecker.java | 257 -
.../parfor/opt/OptTreePlanMapping.java | 75 -
.../parfor/opt/OptTreePlanMappingAbstract.java | 122 -
.../parfor/opt/OptTreePlanMappingRuntime.java | 86 -
.../parfor/opt/OptimizationWrapper.java | 528 --
.../controlprogram/parfor/opt/Optimizer.java | 175 -
.../parfor/opt/OptimizerConstrained.java | 441 --
.../parfor/opt/OptimizerHeuristic.java | 80 -
.../parfor/opt/OptimizerRuleBased.java | 3182 ---------
.../controlprogram/parfor/opt/PerfTestDef.java | 98 -
.../parfor/opt/PerfTestExtFunctCP.java | 92 -
.../parfor/opt/PerfTestMemoryObserver.java | 120 -
.../controlprogram/parfor/opt/PerfTestTool.java | 1751 -----
.../parfor/opt/PerfTestToolRegression.dml | 55 -
.../parfor/opt/ProgramRecompiler.java | 721 --
.../parfor/stat/InfrastructureAnalyzer.java | 551 --
.../controlprogram/parfor/stat/Stat.java | 60 -
.../parfor/stat/StatisticMonitor.java | 245 -
.../controlprogram/parfor/stat/Timing.java | 81 -
.../controlprogram/parfor/util/Cell.java | 70 -
.../controlprogram/parfor/util/IDHandler.java | 186 -
.../controlprogram/parfor/util/IDSequence.java | 79 -
.../parfor/util/PairWritableBlock.java | 59 -
.../parfor/util/PairWritableCell.java | 59 -
.../parfor/util/StagingFileUtils.java | 257 -
.../ibm/bi/dml/runtime/functionobjects/And.java | 49 -
.../bi/dml/runtime/functionobjects/Builtin.java | 528 --
.../ibm/bi/dml/runtime/functionobjects/CM.java | 467 --
.../ibm/bi/dml/runtime/functionobjects/COV.java | 147 -
.../bi/dml/runtime/functionobjects/CTable.java | 184 -
.../dml/runtime/functionobjects/DiagIndex.java | 78 -
.../bi/dml/runtime/functionobjects/Divide.java | 62 -
.../bi/dml/runtime/functionobjects/Equals.java | 90 -
.../runtime/functionobjects/FileFunction.java | 23 -
.../runtime/functionobjects/FunctionObject.java | 149 -
.../runtime/functionobjects/GreaterThan.java | 85 -
.../functionobjects/GreaterThanEquals.java | 85 -
.../runtime/functionobjects/IndexFunction.java | 27 -
.../runtime/functionobjects/IntegerDivide.java | 86 -
.../runtime/functionobjects/KahanFunction.java | 43 -
.../dml/runtime/functionobjects/KahanPlus.java | 110 -
.../runtime/functionobjects/KahanPlusSq.java | 101 -
.../dml/runtime/functionobjects/LessThan.java | 85 -
.../runtime/functionobjects/LessThanEquals.java | 85 -
.../dml/runtime/functionobjects/MaxIndex.java | 75 -
.../bi/dml/runtime/functionobjects/Mean.java | 69 -
.../dml/runtime/functionobjects/MinIndex.java | 75 -
.../bi/dml/runtime/functionobjects/Minus.java | 69 -
.../runtime/functionobjects/Minus1Multiply.java | 68 -
.../bi/dml/runtime/functionobjects/MinusNz.java | 58 -
.../bi/dml/runtime/functionobjects/Modulus.java | 79 -
.../dml/runtime/functionobjects/Multiply.java | 69 -
.../dml/runtime/functionobjects/Multiply2.java | 71 -
.../ibm/bi/dml/runtime/functionobjects/Not.java | 46 -
.../dml/runtime/functionobjects/NotEquals.java | 89 -
.../functionobjects/OffsetColumnIndex.java | 81 -
.../ibm/bi/dml/runtime/functionobjects/Or.java | 49 -
.../functionobjects/ParameterizedBuiltin.java | 310 -
.../bi/dml/runtime/functionobjects/Plus.java | 77 -
.../bi/dml/runtime/functionobjects/Power.java | 64 -
.../bi/dml/runtime/functionobjects/Power2.java | 74 -
.../dml/runtime/functionobjects/ReduceAll.java | 73 -
.../dml/runtime/functionobjects/ReduceCol.java | 73 -
.../dml/runtime/functionobjects/ReduceDiag.java | 73 -
.../dml/runtime/functionobjects/ReduceRow.java | 75 -
.../dml/runtime/functionobjects/RemoveFile.java | 56 -
.../dml/runtime/functionobjects/RenameFile.java | 56 -
.../dml/runtime/functionobjects/SortIndex.java | 77 -
.../dml/runtime/functionobjects/SwapIndex.java | 73 -
.../runtime/functionobjects/ValueFunction.java | 25 -
.../instructions/CPInstructionParser.java | 402 --
.../dml/runtime/instructions/Instruction.java | 233 -
.../runtime/instructions/InstructionParser.java | 79 -
.../runtime/instructions/InstructionUtils.java | 809 ---
.../instructions/MRInstructionParser.java | 619 --
.../runtime/instructions/MRJobInstruction.java | 1521 -----
.../instructions/SPInstructionParser.java | 416 --
.../cp/AggregateBinaryCPInstruction.java | 89 -
.../cp/AggregateTernaryCPInstruction.java | 91 -
.../cp/AggregateUnaryCPInstruction.java | 159 -
.../instructions/cp/AppendCPInstruction.java | 140 -
.../cp/ArithmeticBinaryCPInstruction.java | 112 -
.../instructions/cp/BinaryCPInstruction.java | 67 -
.../cp/BooleanBinaryCPInstruction.java | 71 -
.../runtime/instructions/cp/BooleanObject.java | 73 -
.../cp/BooleanUnaryCPInstruction.java | 69 -
.../instructions/cp/BreakPointInstruction.java | 122 -
.../cp/BuiltinBinaryCPInstruction.java | 67 -
.../cp/BuiltinUnaryCPInstruction.java | 81 -
.../runtime/instructions/cp/CM_COV_Object.java | 204 -
.../runtime/instructions/cp/CPInstruction.java | 92 -
.../dml/runtime/instructions/cp/CPOperand.java | 116 -
.../cp/CentralMomentCPInstruction.java | 132 -
.../cp/ComputationCPInstruction.java | 75 -
.../cp/CovarianceCPInstruction.java | 120 -
.../bi/dml/runtime/instructions/cp/Data.java | 80 -
.../instructions/cp/DataGenCPInstruction.java | 324 -
.../cp/DataPartitionCPInstruction.java | 112 -
.../runtime/instructions/cp/DoubleObject.java | 73 -
.../instructions/cp/FileCPInstruction.java | 130 -
.../dml/runtime/instructions/cp/FileObject.java | 69 -
.../cp/FunctionCallCPInstruction.java | 294 -
.../dml/runtime/instructions/cp/IntObject.java | 75 -
.../runtime/instructions/cp/KahanObject.java | 101 -
.../instructions/cp/MMChainCPInstruction.java | 107 -
.../instructions/cp/MMTSJCPInstruction.java | 90 -
.../cp/MatrixBuiltinCPInstruction.java | 62 -
.../cp/MatrixIndexingCPInstruction.java | 197 -
.../cp/MatrixMatrixArithmeticCPInstruction.java | 64 -
.../cp/MatrixMatrixBuiltinCPInstruction.java | 73 -
.../cp/MatrixMatrixRelationalCPInstruction.java | 62 -
.../cp/MatrixReshapeCPInstruction.java | 95 -
.../cp/MatrixScalarBuiltinCPInstruction.java | 65 -
.../cp/MultiReturnBuiltinCPInstruction.java | 116 -
.../instructions/cp/PMMJCPInstruction.java | 86 -
.../cp/ParameterizedBuiltinCPInstruction.java | 245 -
.../cp/QuantilePickCPInstruction.java | 222 -
.../cp/QuantileSortCPInstruction.java | 106 -
.../cp/QuaternaryCPInstruction.java | 145 -
.../cp/RelationalBinaryCPInstruction.java | 99 -
.../instructions/cp/ReorgCPInstruction.java | 135 -
.../cp/ScalarBuiltinCPInstruction.java | 82 -
.../cp/ScalarMatrixArithmeticCPInstruction.java | 65 -
.../cp/ScalarMatrixRelationalCPInstruction.java | 65 -
.../runtime/instructions/cp/ScalarObject.java | 49 -
.../cp/ScalarScalarArithmeticCPInstruction.java | 89 -
.../cp/ScalarScalarBuiltinCPInstruction.java | 92 -
.../cp/ScalarScalarRelationalCPInstruction.java | 76 -
.../cp/StringInitCPInstruction.java | 117 -
.../runtime/instructions/cp/StringObject.java | 91 -
.../instructions/cp/TernaryCPInstruction.java | 182 -
.../cp/UaggOuterChainCPInstruction.java | 129 -
.../instructions/cp/UnaryCPInstruction.java | 99 -
.../instructions/cp/VariableCPInstruction.java | 1021 ---
.../cpfile/MatrixIndexingCPFileInstruction.java | 153 -
.../ParameterizedBuiltinCPFileInstruction.java | 1168 ----
.../mr/AggregateBinaryInstruction.java | 270 -
.../instructions/mr/AggregateInstruction.java | 75 -
.../mr/AggregateUnaryInstruction.java | 116 -
.../instructions/mr/AppendGInstruction.java | 191 -
.../instructions/mr/AppendInstruction.java | 83 -
.../instructions/mr/AppendMInstruction.java | 153 -
.../instructions/mr/AppendRInstruction.java | 74 -
.../mr/BinUaggChainInstruction.java | 124 -
.../instructions/mr/BinaryInstruction.java | 112 -
.../instructions/mr/BinaryMInstruction.java | 126 -
.../mr/BinaryMRInstructionBase.java | 44 -
.../instructions/mr/CM_N_COVInstruction.java | 93 -
.../instructions/mr/CSVReblockInstruction.java | 80 -
.../instructions/mr/CSVWriteInstruction.java | 69 -
.../mr/CombineBinaryInstruction.java | 77 -
.../mr/CombineTernaryInstruction.java | 66 -
.../mr/CombineUnaryInstruction.java | 68 -
.../mr/CumulativeAggregateInstruction.java | 106 -
.../mr/CumulativeOffsetInstruction.java | 107 -
.../mr/CumulativeSplitInstruction.java | 112 -
.../instructions/mr/DataGenMRInstruction.java | 83 -
.../mr/DataPartitionMRInstruction.java | 50 -
.../mr/GroupedAggregateInstruction.java | 99 -
.../mr/IDistributedCacheConsumer.java | 45 -
.../instructions/mr/MMTSJMRInstruction.java | 118 -
.../runtime/instructions/mr/MRInstruction.java | 98 -
.../mr/MapMultChainInstruction.java | 267 -
.../mr/MatrixReshapeMRInstruction.java | 131 -
.../instructions/mr/PMMJMRInstruction.java | 156 -
.../mr/ParameterizedBuiltinMRInstruction.java | 181 -
.../instructions/mr/PickByCountInstruction.java | 124 -
.../instructions/mr/QuaternaryInstruction.java | 348 -
.../instructions/mr/RandInstruction.java | 110 -
.../mr/RangeBasedReIndexInstruction.java | 130 -
.../instructions/mr/ReblockInstruction.java | 73 -
.../mr/RemoveEmptyMRInstruction.java | 109 -
.../instructions/mr/ReorgInstruction.java | 162 -
.../instructions/mr/ReplicateInstruction.java | 163 -
.../instructions/mr/ScalarInstruction.java | 115 -
.../runtime/instructions/mr/SeqInstruction.java | 78 -
.../instructions/mr/TernaryInstruction.java | 317 -
.../mr/UaggOuterChainInstruction.java | 246 -
.../instructions/mr/UnaryInstruction.java | 96 -
.../instructions/mr/UnaryMRInstructionBase.java | 43 -
.../instructions/mr/ZeroOutInstruction.java | 116 -
.../spark/AggregateTernarySPInstruction.java | 175 -
.../spark/AggregateUnarySPInstruction.java | 251 -
.../spark/AppendGAlignedSPInstruction.java | 120 -
.../spark/AppendGSPInstruction.java | 251 -
.../spark/AppendMSPInstruction.java | 290 -
.../spark/AppendRSPInstruction.java | 116 -
.../spark/ArithmeticBinarySPInstruction.java | 94 -
.../spark/BinUaggChainSPInstruction.java | 122 -
.../instructions/spark/BinarySPInstruction.java | 363 -
.../spark/BuiltinBinarySPInstruction.java | 81 -
.../spark/BuiltinUnarySPInstruction.java | 41 -
.../spark/CSVReblockSPInstruction.java | 132 -
.../spark/CentralMomentSPInstruction.java | 221 -
.../spark/CheckpointSPInstruction.java | 158 -
.../spark/ComputationSPInstruction.java | 105 -
.../spark/CovarianceSPInstruction.java | 208 -
.../instructions/spark/CpmmSPInstruction.java | 191 -
.../spark/CumulativeAggregateSPInstruction.java | 147 -
.../spark/CumulativeOffsetSPInstruction.java | 220 -
.../spark/MapmmChainSPInstruction.java | 219 -
.../instructions/spark/MapmmSPInstruction.java | 384 --
.../MatrixBVectorArithmeticSPInstruction.java | 55 -
.../MatrixBVectorBuiltinSPInstruction.java | 54 -
.../MatrixBVectorRelationalSPInstruction.java | 55 -
.../spark/MatrixBuiltinSPInstruction.java | 82 -
.../spark/MatrixIndexingSPInstruction.java | 526 --
.../MatrixMatrixArithmeticSPInstruction.java | 51 -
.../spark/MatrixMatrixBuiltinSPInstruction.java | 41 -
.../MatrixMatrixRelationalSPInstruction.java | 50 -
.../spark/MatrixReshapeSPInstruction.java | 154 -
.../MatrixScalarArithmeticSPInstruction.java | 50 -
.../spark/MatrixScalarBuiltinSPInstruction.java | 48 -
.../MatrixScalarRelationalSPInstruction.java | 31 -
.../instructions/spark/PMapmmSPInstruction.java | 223 -
.../ParameterizedBuiltinSPInstruction.java | 587 --
.../instructions/spark/PmmSPInstruction.java | 189 -
.../spark/QuantilePickSPInstruction.java | 293 -
.../spark/QuantileSortSPInstruction.java | 123 -
.../spark/QuaternarySPInstruction.java | 571 --
.../instructions/spark/RandSPInstruction.java | 753 ---
.../spark/ReblockSPInstruction.java | 185 -
.../spark/RelationalBinarySPInstruction.java | 67 -
.../instructions/spark/ReorgSPInstruction.java | 284 -
.../instructions/spark/RmmSPInstruction.java | 205 -
.../instructions/spark/SPInstruction.java | 139 -
.../spark/TernarySPInstruction.java | 523 --
.../instructions/spark/TsmmSPInstruction.java | 118 -
.../spark/UaggOuterChainSPInstruction.java | 400 --
.../instructions/spark/UnarySPInstruction.java | 99 -
.../instructions/spark/WriteSPInstruction.java | 290 -
.../instructions/spark/ZipmmSPInstruction.java | 148 -
.../spark/data/BlockPartitioner.java | 114 -
.../spark/data/BroadcastObject.java | 53 -
.../spark/data/CorrMatrixBlock.java | 139 -
.../instructions/spark/data/CountLinesInfo.java | 38 -
.../spark/data/LazyIterableIterator.java | 88 -
.../instructions/spark/data/LineageObject.java | 81 -
.../spark/data/PartitionedBroadcastMatrix.java | 121 -
.../spark/data/PartitionedMatrixBlock.java | 385 --
.../instructions/spark/data/RDDObject.java | 122 -
.../instructions/spark/data/RDDProperties.java | 55 -
.../instructions/spark/data/RowMatrixBlock.java | 135 -
.../spark/data/SerLongWritable.java | 36 -
.../instructions/spark/data/SerText.java | 36 -
.../AggregateDropCorrectionFunction.java | 50 -
.../ComputeBinaryBlockNnzFunction.java | 45 -
.../ConvertColumnRDDToBinaryBlock.java | 68 -
.../ConvertMLLibBlocksToBinaryBlocks.java | 77 -
.../functions/ConvertMatrixBlockToIJVLines.java | 67 -
.../functions/ConvertStringToLongTextPair.java | 34 -
.../ConvertTextLineToBinaryCellFunction.java | 91 -
.../spark/functions/ConvertTextToString.java | 31 -
.../spark/functions/CopyBinaryCellFunction.java | 43 -
.../spark/functions/CopyBlockFunction.java | 50 -
.../spark/functions/CopyBlockPairFunction.java | 60 -
.../spark/functions/CopyTextInputFunction.java | 41 -
.../functions/ExtractBlockForBinaryReblock.java | 129 -
.../spark/functions/ExtractGroup.java | 91 -
.../spark/functions/ExtractGroupNWeights.java | 120 -
.../functions/FilterDiagBlocksFunction.java | 45 -
.../functions/FilterNonEmptyBlocksFunction.java | 48 -
.../FindMatrixBlockFromMatrixIndexes.java | 44 -
.../spark/functions/GetMIMBFromRow.java | 44 -
.../spark/functions/GetMLBlock.java | 40 -
.../spark/functions/GetMLLibBlocks.java | 121 -
.../spark/functions/IsBlockInRange.java | 52 -
.../spark/functions/LastCellInMatrixBlock.java | 53 -
.../functions/MatrixMatrixBinaryOpFunction.java | 44 -
.../functions/MatrixScalarUnaryFunction.java | 45 -
.../functions/MatrixVectorBinaryOpFunction.java | 63 -
.../MatrixVectorBinaryOpPartitionFunction.java | 84 -
.../functions/OuterVectorBinaryOpFunction.java | 104 -
.../functions/PerformGroupByAggInCombiner.java | 101 -
.../functions/PerformGroupByAggInReducer.java | 110 -
.../spark/functions/ReorgMapFunction.java | 74 -
.../functions/ReplicateVectorFunction.java | 69 -
.../spark/functions/SparkListener.java | 176 -
.../UnflattenIterablesAfterCogroup.java | 39 -
.../spark/utils/RDDAggregateUtils.java | 777 ---
.../spark/utils/RDDConverterUtils.java | 748 ---
.../spark/utils/RDDConverterUtilsExt.java | 715 --
.../instructions/spark/utils/RDDSortUtils.java | 843 ---
.../instructions/spark/utils/SparkUtils.java | 448 --
.../runtime/io/BinaryBlockSerialization.java | 142 -
.../ibm/bi/dml/runtime/io/IOUtilFunctions.java | 203 -
.../com/ibm/bi/dml/runtime/io/MatrixReader.java | 137 -
.../bi/dml/runtime/io/MatrixReaderFactory.java | 115 -
.../com/ibm/bi/dml/runtime/io/MatrixWriter.java | 142 -
.../bi/dml/runtime/io/MatrixWriterFactory.java | 98 -
.../ibm/bi/dml/runtime/io/ReadProperties.java | 46 -
.../bi/dml/runtime/io/ReaderBinaryBlock.java | 259 -
.../runtime/io/ReaderBinaryBlockParallel.java | 223 -
.../ibm/bi/dml/runtime/io/ReaderBinaryCell.java | 140 -
.../ibm/bi/dml/runtime/io/ReaderTextCSV.java | 259 -
.../dml/runtime/io/ReaderTextCSVParallel.java | 533 --
.../ibm/bi/dml/runtime/io/ReaderTextCell.java | 299 -
.../dml/runtime/io/ReaderTextCellParallel.java | 357 -
.../bi/dml/runtime/io/WriterBinaryBlock.java | 417 --
.../runtime/io/WriterBinaryBlockParallel.java | 202 -
.../ibm/bi/dml/runtime/io/WriterBinaryCell.java | 147 -
.../bi/dml/runtime/io/WriterMatrixMarket.java | 236 -
.../runtime/io/WriterMatrixMarketParallel.java | 207 -
.../ibm/bi/dml/runtime/io/WriterTextCSV.java | 442 --
.../dml/runtime/io/WriterTextCSVParallel.java | 274 -
.../ibm/bi/dml/runtime/io/WriterTextCell.java | 158 -
.../dml/runtime/io/WriterTextCellParallel.java | 195 -
.../com/ibm/bi/dml/runtime/matrix/CMCOVMR.java | 134 -
.../ibm/bi/dml/runtime/matrix/CSVReblockMR.java | 473 --
.../ibm/bi/dml/runtime/matrix/CleanupMR.java | 180 -
.../ibm/bi/dml/runtime/matrix/CombineMR.java | 377 --
.../ibm/bi/dml/runtime/matrix/DataGenMR.java | 397 --
.../bi/dml/runtime/matrix/DataPartitionMR.java | 100 -
.../java/com/ibm/bi/dml/runtime/matrix/GMR.java | 424 --
.../ibm/bi/dml/runtime/matrix/GroupedAggMR.java | 144 -
.../ibm/bi/dml/runtime/matrix/JobReturn.java | 90 -
.../com/ibm/bi/dml/runtime/matrix/MMCJMR.java | 303 -
.../com/ibm/bi/dml/runtime/matrix/MMRJMR.java | 199 -
.../runtime/matrix/MatrixCharacteristics.java | 423 --
.../matrix/MatrixDimensionsMetaData.java | 76 -
.../runtime/matrix/MatrixFormatMetaData.java | 51 -
.../com/ibm/bi/dml/runtime/matrix/MetaData.java | 44 -
.../ibm/bi/dml/runtime/matrix/ReblockMR.java | 228 -
.../com/ibm/bi/dml/runtime/matrix/SortMR.java | 414 --
.../ibm/bi/dml/runtime/matrix/WriteCSVMR.java | 190 -
.../matrix/data/AdaptivePartialBlock.java | 160 -
.../matrix/data/AddDummyWeightConverter.java | 81 -
.../data/BinaryBlockToBinaryCellConverter.java | 137 -
.../data/BinaryBlockToRowBlockConverter.java | 108 -
.../data/BinaryBlockToTextCellConverter.java | 142 -
.../data/BinaryCellToRowBlockConverter.java | 57 -
.../matrix/data/BinaryCellToTextConverter.java | 57 -
.../dml/runtime/matrix/data/CM_N_COVCell.java | 382 --
.../matrix/data/CSVFileFormatProperties.java | 110 -
.../bi/dml/runtime/matrix/data/CTableMap.java | 139 -
.../bi/dml/runtime/matrix/data/Converter.java | 30 -
.../matrix/data/FileFormatProperties.java | 43 -
.../com/ibm/bi/dml/runtime/matrix/data/IJV.java | 52 -
.../runtime/matrix/data/IdenticalConverter.java | 51 -
.../bi/dml/runtime/matrix/data/InputInfo.java | 159 -
.../dml/runtime/matrix/data/LibCommonsMath.java | 244 -
.../dml/runtime/matrix/data/LibMatrixAgg.java | 2863 --------
.../runtime/matrix/data/LibMatrixBincell.java | 1320 ----
.../runtime/matrix/data/LibMatrixDatagen.java | 806 ---
.../dml/runtime/matrix/data/LibMatrixMult.java | 4285 ------------
.../runtime/matrix/data/LibMatrixOuterAgg.java | 1599 -----
.../dml/runtime/matrix/data/LibMatrixReorg.java | 2116 ------
.../bi/dml/runtime/matrix/data/MatrixBlock.java | 6295 ------------------
.../matrix/data/MatrixBlockDataInput.java | 57 -
.../matrix/data/MatrixBlockDataOutput.java | 55 -
.../bi/dml/runtime/matrix/data/MatrixCell.java | 491 --
.../dml/runtime/matrix/data/MatrixIndexes.java | 198 -
.../runtime/matrix/data/MatrixPackedCell.java | 197 -
.../bi/dml/runtime/matrix/data/MatrixValue.java | 212 -
.../matrix/data/MultipleOutputCommitter.java | 151 -
.../data/NumItemsByEachReducerMetaData.java | 67 -
.../matrix/data/OperationsOnMatrixValues.java | 351 -
.../bi/dml/runtime/matrix/data/OutputInfo.java | 150 -
.../ibm/bi/dml/runtime/matrix/data/Pair.java | 63 -
.../dml/runtime/matrix/data/PartialBlock.java | 103 -
.../data/PoissonRandomMatrixGenerator.java | 49 -
.../matrix/data/RandomMatrixGenerator.java | 101 -
.../runtime/matrix/data/RuntimeDataFormat.java | 61 -
.../bi/dml/runtime/matrix/data/SparseRow.java | 524 --
.../runtime/matrix/data/SparseRowsIterator.java | 108 -
.../ibm/bi/dml/runtime/matrix/data/Tagged.java | 120 -
.../matrix/data/TaggedAdaptivePartialBlock.java | 34 -
.../matrix/data/TaggedFirstSecondIndexes.java | 216 -
.../bi/dml/runtime/matrix/data/TaggedInt.java | 59 -
.../runtime/matrix/data/TaggedMatrixBlock.java | 39 -
.../runtime/matrix/data/TaggedMatrixCell.java | 39 -
.../matrix/data/TaggedMatrixPackedCell.java | 39 -
.../runtime/matrix/data/TaggedMatrixValue.java | 48 -
.../runtime/matrix/data/TaggedPartialBlock.java | 33 -
.../matrix/data/TaggedTripleIndexes.java | 154 -
.../data/TextCellToRowBlockConverter.java | 83 -
.../matrix/data/TextToBinaryCellConverter.java | 84 -
.../dml/runtime/matrix/data/TripleIndexes.java | 164 -
.../matrix/data/UnPaddedOutputFormat.java | 64 -
.../dml/runtime/matrix/data/WeightedCell.java | 191 -
.../data/WeightedCellToSortInputConverter.java | 58 -
.../dml/runtime/matrix/data/WeightedPair.java | 110 -
.../data/WeightedPairToSortInputConverter.java | 58 -
.../data/hadoopfix/DelegatingInputFormat.java | 137 -
.../matrix/data/hadoopfix/MultipleInputs.java | 133 -
.../matrix/data/hadoopfix/TaggedInputSplit.java | 141 -
.../runtime/matrix/mapred/CMCOVMRMapper.java | 161 -
.../runtime/matrix/mapred/CMCOVMRReducer.java | 153 -
.../matrix/mapred/CSVAssignRowIDMapper.java | 157 -
.../matrix/mapred/CSVAssignRowIDReducer.java | 63 -
.../runtime/matrix/mapred/CSVReblockMapper.java | 200 -
.../matrix/mapred/CSVReblockReducer.java | 124 -
.../runtime/matrix/mapred/CSVWriteMapper.java | 108 -
.../runtime/matrix/mapred/CSVWriteReducer.java | 384 --
.../bi/dml/runtime/matrix/mapred/CachedMap.java | 133 -
.../runtime/matrix/mapred/CachedMapElement.java | 27 -
.../runtime/matrix/mapred/CachedValueMap.java | 150 -
.../mapred/CollectMultipleConvertedOutputs.java | 76 -
.../runtime/matrix/mapred/DataGenMapper.java | 136 -
.../matrix/mapred/DistributedCacheInput.java | 227 -
.../dml/runtime/matrix/mapred/GMRCombiner.java | 66 -
.../runtime/matrix/mapred/GMRCtableBuffer.java | 237 -
.../bi/dml/runtime/matrix/mapred/GMRMapper.java | 271 -
.../dml/runtime/matrix/mapred/GMRReducer.java | 173 -
.../matrix/mapred/GroupedAggMRCombiner.java | 167 -
.../matrix/mapred/GroupedAggMRMapper.java | 123 -
.../matrix/mapred/GroupedAggMRReducer.java | 153 -
.../matrix/mapred/IndexedMatrixValue.java | 85 -
.../dml/runtime/matrix/mapred/MMCJMRCache.java | 161 -
.../runtime/matrix/mapred/MMCJMRCombiner.java | 53 -
.../mapred/MMCJMRCombinerReducerBase.java | 143 -
.../runtime/matrix/mapred/MMCJMRInputCache.java | 160 -
.../dml/runtime/matrix/mapred/MMCJMRMapper.java | 132 -
.../runtime/matrix/mapred/MMCJMRReducer.java | 303 -
.../mapred/MMCJMRReducerWithAggregator.java | 251 -
.../dml/runtime/matrix/mapred/MMRJMRMapper.java | 139 -
.../runtime/matrix/mapred/MMRJMRReducer.java | 165 -
.../mapred/MRBaseForCommonInstructions.java | 337 -
.../matrix/mapred/MRConfigurationNames.java | 68 -
.../matrix/mapred/MRJobConfiguration.java | 1909 ------
.../dml/runtime/matrix/mapred/MapperBase.java | 433 --
.../matrix/mapred/PartialAggregator.java | 258 -
.../runtime/matrix/mapred/ReblockBuffer.java | 408 --
.../runtime/matrix/mapred/ReblockMapper.java | 228 -
.../runtime/matrix/mapred/ReblockReducer.java | 156 -
.../dml/runtime/matrix/mapred/ReduceBase.java | 450 --
.../operators/AggregateBinaryOperator.java | 60 -
.../matrix/operators/AggregateOperator.java | 63 -
.../operators/AggregateUnaryOperator.java | 70 -
.../matrix/operators/BinaryOperator.java | 101 -
.../runtime/matrix/operators/CMOperator.java | 120 -
.../runtime/matrix/operators/COVOperator.java | 36 -
.../matrix/operators/LeftScalarOperator.java | 66 -
.../dml/runtime/matrix/operators/Operator.java | 39 -
.../matrix/operators/QuaternaryOperator.java | 97 -
.../matrix/operators/ReIndexOperator.java | 30 -
.../runtime/matrix/operators/ReorgOperator.java | 39 -
.../matrix/operators/RightScalarOperator.java | 58 -
.../matrix/operators/ScalarOperator.java | 99 -
.../matrix/operators/SimpleOperator.java | 36 -
.../runtime/matrix/operators/UnaryOperator.java | 47 -
.../matrix/operators/ZeroOutOperator.java | 30 -
.../sort/CompactDoubleIntInputFormat.java | 211 -
.../runtime/matrix/sort/CompactInputFormat.java | 122 -
.../matrix/sort/CompactOutputFormat.java | 90 -
.../matrix/sort/IndexSortComparable.java | 82 -
.../matrix/sort/IndexSortComparableDesc.java | 47 -
.../runtime/matrix/sort/IndexSortMapper.java | 73 -
.../runtime/matrix/sort/IndexSortReducer.java | 101 -
.../matrix/sort/IndexSortStitchupMapper.java | 139 -
.../matrix/sort/IndexSortStitchupReducer.java | 72 -
.../matrix/sort/PickFromCompactInputFormat.java | 545 --
.../dml/runtime/matrix/sort/ReadWithZeros.java | 82 -
.../matrix/sort/SamplingSortMRInputFormat.java | 237 -
.../runtime/matrix/sort/ValueSortMapper.java | 95 -
.../runtime/matrix/sort/ValueSortReducer.java | 66 -
.../bi/dml/runtime/transform/ApplyTfBBMR.java | 155 -
.../dml/runtime/transform/ApplyTfBBMapper.java | 149 -
.../bi/dml/runtime/transform/ApplyTfCSVMR.java | 126 -
.../dml/runtime/transform/ApplyTfCSVMapper.java | 110 -
.../dml/runtime/transform/ApplyTfCSVSPARK.java | 158 -
.../ibm/bi/dml/runtime/transform/BinAgent.java | 353 -
.../bi/dml/runtime/transform/DataTransform.java | 1496 -----
.../bi/dml/runtime/transform/DistinctValue.java | 106 -
.../dml/runtime/transform/DummycodeAgent.java | 424 --
.../bi/dml/runtime/transform/GTFMTDMapper.java | 105 -
.../bi/dml/runtime/transform/GTFMTDReducer.java | 122 -
.../bi/dml/runtime/transform/GenTfMtdMR.java | 104 -
.../bi/dml/runtime/transform/GenTfMtdSPARK.java | 233 -
.../bi/dml/runtime/transform/MVImputeAgent.java | 975 ---
.../ibm/bi/dml/runtime/transform/OmitAgent.java | 122 -
.../bi/dml/runtime/transform/RecodeAgent.java | 463 --
.../ibm/bi/dml/runtime/transform/TfUtils.java | 547 --
.../runtime/transform/TransformationAgent.java | 91 -
.../runtime/util/BinaryBlockInputFormat.java | 45 -
.../runtime/util/BinaryBlockRecordReader.java | 67 -
.../ibm/bi/dml/runtime/util/DataConverter.java | 691 --
.../util/FastBufferedDataInputStream.java | 288 -
.../util/FastBufferedDataOutputStream.java | 325 -
.../dml/runtime/util/FastStringTokenizer.java | 162 -
.../com/ibm/bi/dml/runtime/util/IndexRange.java | 48 -
.../ibm/bi/dml/runtime/util/LocalFileUtils.java | 558 --
.../dml/runtime/util/LongLongDoubleHashMap.java | 169 -
.../ibm/bi/dml/runtime/util/MapReduceTool.java | 725 --
.../bi/dml/runtime/util/NormalPRNGenerator.java | 72 -
.../ibm/bi/dml/runtime/util/PRNGenerator.java | 38 -
.../dml/runtime/util/PoissonPRNGenerator.java | 82 -
.../java/com/ibm/bi/dml/runtime/util/RandN.java | 65 -
.../com/ibm/bi/dml/runtime/util/RandNPair.java | 51 -
.../com/ibm/bi/dml/runtime/util/SortUtils.java | 370 -
.../dml/runtime/util/UniformPRNGenerator.java | 49 -
.../ibm/bi/dml/runtime/util/UtilFunctions.java | 333 -
.../java/com/ibm/bi/dml/udf/BinaryObject.java | 50 -
.../ExternalFunctionInvocationInstruction.java | 81 -
.../com/ibm/bi/dml/udf/FunctionParameter.java | 62 -
src/main/java/com/ibm/bi/dml/udf/Matrix.java | 233 -
.../com/ibm/bi/dml/udf/PackageFunction.java | 184 -
.../ibm/bi/dml/udf/PackageRuntimeException.java | 43 -
src/main/java/com/ibm/bi/dml/udf/Scalar.java | 69 -
.../com/ibm/bi/dml/udf/lib/BinningWrapper.java | 109 -
.../com/ibm/bi/dml/udf/lib/DeNaNWrapper.java | 77 -
.../bi/dml/udf/lib/DeNegInfinityWrapper.java | 77 -
.../bi/dml/udf/lib/DynamicProjectMatrixCP.java | 103 -
.../ibm/bi/dml/udf/lib/DynamicReadMatrixCP.java | 84 -
.../bi/dml/udf/lib/DynamicReadMatrixRcCP.java | 95 -
.../bi/dml/udf/lib/DynamicWriteMatrixCP.java | 82 -
.../com/ibm/bi/dml/udf/lib/GatherWrapper.java | 96 -
.../com/ibm/bi/dml/udf/lib/OrderWrapper.java | 123 -
.../dml/udf/lib/PermutationMatrixWrapper.java | 153 -
.../com/ibm/bi/dml/udf/lib/RemoveEmptyRows.java | 153 -
.../com/ibm/bi/dml/udf/lib/TimeWrapper.java | 74 -
.../java/com/ibm/bi/dml/utils/AppException.java | 38 -
src/main/java/com/ibm/bi/dml/utils/Explain.java | 1155 ----
.../InstallDependencyForIntegrationTests.java | 145 -
.../java/com/ibm/bi/dml/utils/JSONHelper.java | 53 -
.../com/ibm/bi/dml/utils/ParameterBuilder.java | 110 -
.../java/com/ibm/bi/dml/utils/Statistics.java | 549 --
src/main/java/com/ibm/bi/dml/utils/Timer.java | 60 -
.../java/com/ibm/bi/dml/yarn/DMLAppMaster.java | 177 -
.../bi/dml/yarn/DMLAppMasterStatusReporter.java | 68 -
.../com/ibm/bi/dml/yarn/DMLAppMasterUtils.java | 223 -
.../java/com/ibm/bi/dml/yarn/DMLYarnClient.java | 588 --
.../com/ibm/bi/dml/yarn/DMLYarnClientProxy.java | 114 -
.../ibm/bi/dml/yarn/ropt/GridEnumeration.java | 52 -
.../bi/dml/yarn/ropt/GridEnumerationEqui.java | 61 -
.../bi/dml/yarn/ropt/GridEnumerationExp.java | 62 -
.../bi/dml/yarn/ropt/GridEnumerationHybrid.java | 61 -
.../bi/dml/yarn/ropt/GridEnumerationMemory.java | 178 -
.../dml/yarn/ropt/MRJobResourceInstruction.java | 50 -
.../ibm/bi/dml/yarn/ropt/ResourceConfig.java | 214 -
.../ibm/bi/dml/yarn/ropt/ResourceOptimizer.java | 800 ---
.../yarn/ropt/ResourceOptimizerCPMigration.java | 265 -
.../bi/dml/yarn/ropt/YarnClusterAnalyzer.java | 742 ---
.../ibm/bi/dml/yarn/ropt/YarnClusterConfig.java | 80 -
.../bi/dml/yarn/ropt/YarnOptimizerUtils.java | 112 -
.../java/org/apache/sysml/api/DMLException.java | 58 +
.../java/org/apache/sysml/api/DMLScript.java | 990 +++
src/main/java/org/apache/sysml/api/MLBlock.java | 241 +
.../java/org/apache/sysml/api/MLContext.java | 1396 ++++
.../org/apache/sysml/api/MLContextProxy.java | 102 +
.../java/org/apache/sysml/api/MLMatrix.java | 408 ++
.../java/org/apache/sysml/api/MLOutput.java | 401 ++
.../org/apache/sysml/api/jmlc/Connection.java | 244 +
.../org/apache/sysml/api/jmlc/JMLCUtils.java | 107 +
.../apache/sysml/api/jmlc/PreparedScript.java | 231 +
.../apache/sysml/api/jmlc/ResultVariables.java | 92 +
.../apache/sysml/api/ml/LogisticRegression.java | 464 ++
.../sysml/api/ml/LogisticRegressionModel.java | 169 +
.../functions/ConvertSingleColumnToString.java | 15 +
.../api/ml/functions/ConvertVectorToDouble.java | 16 +
.../api/monitoring/InstructionComparator.java | 38 +
.../apache/sysml/api/monitoring/Location.java | 77 +
.../api/monitoring/SparkMonitoringUtil.java | 602 ++
.../org/apache/sysml/api/python/SystemML.py | 198 +
.../apache/sysml/conf/ConfigurationManager.java | 79 +
.../java/org/apache/sysml/conf/DMLConfig.java | 491 ++
.../sysml/debug/DMLBreakpointManager.java | 191 +
.../org/apache/sysml/debug/DMLDebugger.java | 609 ++
.../sysml/debug/DMLDebuggerException.java | 41 +
.../sysml/debug/DMLDebuggerFunctions.java | 678 ++
.../sysml/debug/DMLDebuggerInterface.java | 316 +
.../sysml/debug/DMLDebuggerProgramInfo.java | 375 ++
.../org/apache/sysml/debug/DMLDisassemble.java | 201 +
.../java/org/apache/sysml/debug/DMLFrame.java | 68 +
.../apache/sysml/debug/DMLProgramCounter.java | 208 +
.../java/org/apache/sysml/debug/DebugState.java | 190 +
.../java/org/apache/sysml/hops/AggBinaryOp.java | 1959 ++++++
.../java/org/apache/sysml/hops/AggUnaryOp.java | 735 ++
.../java/org/apache/sysml/hops/BinaryOp.java | 1565 +++++
.../java/org/apache/sysml/hops/DataGenOp.java | 532 ++
src/main/java/org/apache/sysml/hops/DataOp.java | 529 ++
.../java/org/apache/sysml/hops/FunctionOp.java | 297 +
src/main/java/org/apache/sysml/hops/Hop.java | 1882 ++++++
.../org/apache/sysml/hops/HopsException.java | 58 +
.../java/org/apache/sysml/hops/IndexingOp.java | 464 ++
.../org/apache/sysml/hops/LeftIndexingOp.java | 503 ++
.../java/org/apache/sysml/hops/LiteralOp.java | 291 +
.../java/org/apache/sysml/hops/MemoTable.java | 243 +
.../org/apache/sysml/hops/OptimizerUtils.java | 1407 ++++
.../sysml/hops/ParameterizedBuiltinOp.java | 1139 ++++
.../org/apache/sysml/hops/QuaternaryOp.java | 1684 +++++
.../java/org/apache/sysml/hops/ReorgOp.java | 608 ++
.../java/org/apache/sysml/hops/TernaryOp.java | 1024 +++
.../java/org/apache/sysml/hops/UnaryOp.java | 746 +++
.../sysml/hops/cost/CostEstimationWrapper.java | 161 +
.../apache/sysml/hops/cost/CostEstimator.java | 787 +++
.../sysml/hops/cost/CostEstimatorNumMRJobs.java | 44 +
.../hops/cost/CostEstimatorStaticRuntime.java | 1317 ++++
.../org/apache/sysml/hops/cost/VarStats.java | 77 +
.../sysml/hops/globalopt/GDFEnumOptimizer.java | 603 ++
.../sysml/hops/globalopt/GlobalOptimizer.java | 47 +
.../hops/globalopt/GlobalOptimizerWrapper.java | 135 +
.../hops/globalopt/InterestingProperties.java | 118 +
.../sysml/hops/globalopt/MemoStructure.java | 94 +
.../org/apache/sysml/hops/globalopt/Plan.java | 228 +
.../apache/sysml/hops/globalopt/PlanSet.java | 168 +
.../sysml/hops/globalopt/RewriteConfig.java | 96 +
.../apache/sysml/hops/globalopt/Summary.java | 99 +
.../globalopt/gdfgraph/GDFCrossBlockNode.java | 93 +
.../sysml/hops/globalopt/gdfgraph/GDFGraph.java | 47 +
.../hops/globalopt/gdfgraph/GDFLoopNode.java | 64 +
.../sysml/hops/globalopt/gdfgraph/GDFNode.java | 174 +
.../hops/globalopt/gdfgraph/GraphBuilder.java | 350 +
.../gdfresolve/GDFMismatchHeuristic.java | 46 +
.../GDFMismatchHeuristicBlocksizeOrFirst.java | 47 +
.../gdfresolve/GDFMismatchHeuristicFirst.java | 36 +
.../gdfresolve/MismatchHeuristicFactory.java | 45 +
.../sysml/hops/ipa/InterProceduralAnalysis.java | 1248 ++++
.../hops/recompile/LiteralReplacement.java | 538 ++
.../sysml/hops/recompile/RecompileStatus.java | 61 +
.../apache/sysml/hops/recompile/Recompiler.java | 2126 ++++++
.../sysml/hops/rewrite/HopDagValidator.java | 126 +
.../sysml/hops/rewrite/HopRewriteRule.java | 60 +
.../sysml/hops/rewrite/HopRewriteUtils.java | 1071 +++
.../hops/rewrite/ProgramRewriteStatus.java | 69 +
.../sysml/hops/rewrite/ProgramRewriter.java | 397 ++
.../RewriteAlgebraicSimplificationDynamic.java | 2506 +++++++
.../RewriteAlgebraicSimplificationStatic.java | 1544 +++++
.../rewrite/RewriteBlockSizeAndReblock.java | 242 +
.../RewriteCommonSubexpressionElimination.java | 232 +
.../hops/rewrite/RewriteConstantFolding.java | 333 +
.../rewrite/RewriteForLoopVectorization.java | 398 ++
.../rewrite/RewriteIndexingVectorization.java | 359 +
.../RewriteInjectSparkLoopCheckpointing.java | 121 +
.../RewriteInjectSparkPReadCheckpointing.java | 94 +
.../RewriteMatrixMultChainOptimization.java | 423 ++
.../RewriteRemovePersistentReadWrite.java | 117 +
.../rewrite/RewriteRemoveReadAfterWrite.java | 124 +
.../RewriteRemoveUnnecessaryBranches.java | 82 +
.../rewrite/RewriteRemoveUnnecessaryCasts.java | 120 +
.../RewriteSplitDagDataDependentOperators.java | 519 ++
.../rewrite/RewriteSplitDagUnknownCSVRead.java | 172 +
.../RewriteTransientWriteParentHandling.java | 89 +
.../hops/rewrite/StatementBlockRewriteRule.java | 50 +
.../java/org/apache/sysml/lops/Aggregate.java | 190 +
.../java/org/apache/sysml/lops/AppendCP.java | 91 +
.../java/org/apache/sysml/lops/AppendG.java | 128 +
.../org/apache/sysml/lops/AppendGAlignedSP.java | 74 +
.../java/org/apache/sysml/lops/AppendM.java | 132 +
.../java/org/apache/sysml/lops/AppendR.java | 104 +
src/main/java/org/apache/sysml/lops/Binary.java | 194 +
.../java/org/apache/sysml/lops/BinaryM.java | 218 +
.../org/apache/sysml/lops/BinaryScalar.java | 199 +
.../org/apache/sysml/lops/BinaryUAggChain.java | 145 +
.../java/org/apache/sysml/lops/CSVReBlock.java | 193 +
.../org/apache/sysml/lops/CentralMoment.java | 162 +
.../java/org/apache/sysml/lops/Checkpoint.java | 167 +
.../java/org/apache/sysml/lops/CoVariance.java | 179 +
.../org/apache/sysml/lops/CombineBinary.java | 127 +
.../org/apache/sysml/lops/CombineTernary.java | 120 +
.../org/apache/sysml/lops/CombineUnary.java | 99 +
.../sysml/lops/CumulativeOffsetBinary.java | 165 +
.../sysml/lops/CumulativePartialAggregate.java | 137 +
.../sysml/lops/CumulativeSplitAggregate.java | 87 +
src/main/java/org/apache/sysml/lops/Data.java | 619 ++
.../java/org/apache/sysml/lops/DataGen.java | 558 ++
.../org/apache/sysml/lops/DataPartition.java | 115 +
.../org/apache/sysml/lops/FunctionCallCP.java | 150 +
src/main/java/org/apache/sysml/lops/Group.java | 73 +
.../org/apache/sysml/lops/GroupedAggregate.java | 244 +
.../java/org/apache/sysml/lops/LeftIndex.java | 178 +
src/main/java/org/apache/sysml/lops/Lop.java | 720 ++
.../org/apache/sysml/lops/LopProperties.java | 172 +
.../org/apache/sysml/lops/LopsException.java | 40 +
src/main/java/org/apache/sysml/lops/MMCJ.java | 148 +
src/main/java/org/apache/sysml/lops/MMRJ.java | 106 +
src/main/java/org/apache/sysml/lops/MMTSJ.java | 122 +
src/main/java/org/apache/sysml/lops/MMZip.java | 83 +
.../java/org/apache/sysml/lops/MapMult.java | 213 +
.../org/apache/sysml/lops/MapMultChain.java | 268 +
.../org/apache/sysml/lops/OutputParameters.java | 158 +
src/main/java/org/apache/sysml/lops/PMMJ.java | 164 +
.../java/org/apache/sysml/lops/PMapMult.java | 86 +
.../apache/sysml/lops/ParameterizedBuiltin.java | 514 ++
.../org/apache/sysml/lops/PartialAggregate.java | 414 ++
.../java/org/apache/sysml/lops/PickByCount.java | 182 +
.../apache/sysml/lops/RangeBasedReIndex.java | 198 +
.../java/org/apache/sysml/lops/ReBlock.java | 168 +
src/main/java/org/apache/sysml/lops/RepMat.java | 91 +
.../java/org/apache/sysml/lops/SortKeys.java | 183 +
.../java/org/apache/sysml/lops/Ternary.java | 385 ++
.../org/apache/sysml/lops/TernaryAggregate.java | 104 +
.../java/org/apache/sysml/lops/Transform.java | 286 +
.../org/apache/sysml/lops/UAggOuterChain.java | 170 +
src/main/java/org/apache/sysml/lops/Unary.java | 445 ++
.../java/org/apache/sysml/lops/UnaryCP.java | 178 +
.../apache/sysml/lops/WeightedCrossEntropy.java | 156 +
.../sysml/lops/WeightedCrossEntropyR.java | 156 +
.../org/apache/sysml/lops/WeightedDivMM.java | 186 +
.../org/apache/sysml/lops/WeightedDivMMR.java | 155 +
.../org/apache/sysml/lops/WeightedSigmoid.java | 160 +
.../org/apache/sysml/lops/WeightedSigmoidR.java | 156 +
.../apache/sysml/lops/WeightedSquaredLoss.java | 169 +
.../apache/sysml/lops/WeightedSquaredLossR.java | 160 +
.../org/apache/sysml/lops/WeightedUnaryMM.java | 165 +
.../org/apache/sysml/lops/WeightedUnaryMMR.java | 162 +
.../java/org/apache/sysml/lops/ZeroOut.java | 132 +
.../java/org/apache/sysml/lops/compile/Dag.java | 4574 +++++++++++++
.../org/apache/sysml/lops/compile/JobType.java | 252 +
.../sysml/lops/compile/LopComparator.java | 62 +
.../apache/sysml/lops/runtime/RunMRJobs.java | 591 ++
.../org/apache/sysml/parser/AParserWrapper.java | 62 +
.../sysml/parser/AssignmentStatement.java | 146 +
.../apache/sysml/parser/BinaryExpression.java | 227 +
.../apache/sysml/parser/BooleanExpression.java | 159 +
.../apache/sysml/parser/BooleanIdentifier.java | 64 +
.../sysml/parser/BuiltinFunctionExpression.java | 1425 ++++
.../sysml/parser/ConditionalPredicate.java | 92 +
.../apache/sysml/parser/ConstIdentifier.java | 29 +
.../apache/sysml/parser/DMLParseException.java | 87 +
.../org/apache/sysml/parser/DMLProgram.java | 783 +++
.../org/apache/sysml/parser/DMLTranslator.java | 2814 ++++++++
.../org/apache/sysml/parser/DataExpression.java | 2171 ++++++
.../org/apache/sysml/parser/DataIdentifier.java | 174 +
.../apache/sysml/parser/DoubleIdentifier.java | 85 +
.../org/apache/sysml/parser/Expression.java | 513 ++
.../sysml/parser/ExternalFunctionStatement.java | 173 +
.../org/apache/sysml/parser/ForStatement.java | 107 +
.../apache/sysml/parser/ForStatementBlock.java | 444 ++
.../sysml/parser/FunctionCallIdentifier.java | 208 +
.../apache/sysml/parser/FunctionStatement.java | 154 +
.../sysml/parser/FunctionStatementBlock.java | 321 +
.../org/apache/sysml/parser/Identifier.java | 281 +
.../org/apache/sysml/parser/IfStatement.java | 162 +
.../apache/sysml/parser/IfStatementBlock.java | 537 ++
.../apache/sysml/parser/ImportStatement.java | 113 +
.../apache/sysml/parser/IndexedIdentifier.java | 782 +++
.../org/apache/sysml/parser/IntIdentifier.java | 78 +
.../apache/sysml/parser/IterablePredicate.java | 219 +
.../apache/sysml/parser/LanguageException.java | 69 +
.../sysml/parser/LiveVariableAnalysis.java | 99 +
.../sysml/parser/MultiAssignmentStatement.java | 151 +
.../apache/sysml/parser/OutputStatement.java | 233 +
.../apache/sysml/parser/ParForStatement.java | 44 +
.../sysml/parser/ParForStatementBlock.java | 2177 ++++++
.../sysml/parser/ParameterExpression.java | 53 +
.../ParameterizedBuiltinFunctionExpression.java | 615 ++
.../org/apache/sysml/parser/ParseException.java | 87 +
.../org/apache/sysml/parser/PathStatement.java | 70 +
.../org/apache/sysml/parser/PrintStatement.java | 121 +
.../sysml/parser/RelationalExpression.java | 219 +
.../java/org/apache/sysml/parser/Statement.java | 134 +
.../org/apache/sysml/parser/StatementBlock.java | 1063 +++
.../apache/sysml/parser/StringIdentifier.java | 75 +
.../java/org/apache/sysml/parser/Token.java | 131 +
.../org/apache/sysml/parser/VariableSet.java | 93 +
.../org/apache/sysml/parser/WhileStatement.java | 106 +
.../sysml/parser/WhileStatementBlock.java | 334 +
.../sysml/parser/antlr4/DMLParserWrapper.java | 384 ++
.../java/org/apache/sysml/parser/antlr4/Dml.g4 | 199 +
.../antlr4/DmlSyntacticErrorListener.java | 113 +
.../parser/antlr4/DmlSyntacticValidator.java | 1517 +++++
.../antlr4/DmlSyntacticValidatorHelper.java | 102 +
.../sysml/parser/antlr4/ExpressionInfo.java | 29 +
.../sysml/parser/antlr4/StatementInfo.java | 34 +
.../sysml/parser/python/ExpressionInfo.java | 13 +
.../sysml/parser/python/PyDMLParserWrapper.java | 307 +
.../org/apache/sysml/parser/python/Pydml.g4 | 382 ++
.../python/PydmlSyntacticErrorListener.java | 113 +
.../parser/python/PydmlSyntacticValidator.java | 2107 ++++++
.../python/PydmlSyntacticValidatorHelper.java | 123 +
.../sysml/parser/python/StatementInfo.java | 13 +
.../sysml/runtime/DMLRuntimeException.java | 41 +
.../sysml/runtime/DMLScriptException.java | 46 +
.../DMLUnsupportedOperationException.java | 30 +
.../ExternalFunctionProgramBlock.java | 1064 +++
.../ExternalFunctionProgramBlockCP.java | 192 +
.../runtime/controlprogram/ForProgramBlock.java | 288 +
.../controlprogram/FunctionProgramBlock.java | 166 +
.../runtime/controlprogram/IfProgramBlock.java | 276 +
.../controlprogram/LocalVariableMap.java | 231 +
.../controlprogram/ParForProgramBlock.java | 2126 ++++++
.../sysml/runtime/controlprogram/Program.java | 164 +
.../runtime/controlprogram/ProgramBlock.java | 436 ++
.../controlprogram/WhileProgramBlock.java | 233 +
.../controlprogram/caching/ByteBuffer.java | 210 +
.../controlprogram/caching/CacheDataInput.java | 232 +
.../controlprogram/caching/CacheDataOutput.java | 226 +
.../controlprogram/caching/CacheException.java | 48 +
.../caching/CacheIOException.java | 47 +
.../controlprogram/caching/CacheStatistics.java | 299 +
.../caching/CacheStatusException.java | 25 +
.../controlprogram/caching/CacheableData.java | 450 ++
.../controlprogram/caching/LazyWriteBuffer.java | 278 +
.../controlprogram/caching/MatrixObject.java | 1686 +++++
.../controlprogram/caching/PageCache.java | 86 +
.../context/ExecutionContext.java | 522 ++
.../context/ExecutionContextFactory.java | 65 +
.../context/SparkExecutionContext.java | 1155 ++++
.../controlprogram/parfor/DataPartitioner.java | 236 +
.../parfor/DataPartitionerLocal.java | 888 +++
.../parfor/DataPartitionerRemoteMR.java | 214 +
.../parfor/DataPartitionerRemoteMapper.java | 707 ++
.../parfor/DataPartitionerRemoteReducer.java | 231 +
.../parfor/DataPartitionerRemoteSpark.java | 125 +
.../DataPartitionerRemoteSparkMapper.java | 153 +
.../DataPartitionerRemoteSparkReducer.java | 87 +
.../controlprogram/parfor/LocalParWorker.java | 148 +
.../controlprogram/parfor/LocalTaskQueue.java | 144 +
.../controlprogram/parfor/ParForBody.java | 85 +
.../controlprogram/parfor/ParWorker.java | 258 +
.../controlprogram/parfor/ProgramConverter.java | 2368 +++++++
.../controlprogram/parfor/RemoteDPParForMR.java | 290 +
.../parfor/RemoteDPParForSpark.java | 119 +
.../parfor/RemoteDPParForSparkWorker.java | 316 +
.../parfor/RemoteDPParWorkerReducer.java | 370 +
.../parfor/RemoteParForColocatedFileSplit.java | 183 +
.../RemoteParForColocatedNLineInputFormat.java | 69 +
.../parfor/RemoteParForJobReturn.java | 87 +
.../controlprogram/parfor/RemoteParForMR.java | 311 +
.../parfor/RemoteParForSpark.java | 106 +
.../parfor/RemoteParForSparkWorker.java | 131 +
.../parfor/RemoteParForUtils.java | 264 +
.../parfor/RemoteParWorkerMapper.java | 258 +
.../controlprogram/parfor/ResultMerge.java | 176 +
.../parfor/ResultMergeLocalAutomatic.java | 78 +
.../parfor/ResultMergeLocalFile.java | 1174 ++++
.../parfor/ResultMergeLocalMemory.java | 338 +
.../parfor/ResultMergeRemoteGrouping.java | 43 +
.../parfor/ResultMergeRemoteMR.java | 339 +
.../parfor/ResultMergeRemoteMapper.java | 199 +
.../parfor/ResultMergeRemotePartitioning.java | 63 +
.../parfor/ResultMergeRemoteReducer.java | 359 +
.../parfor/ResultMergeRemoteSorting.java | 49 +
.../parfor/ResultMergeRemoteSpark.java | 219 +
.../parfor/ResultMergeRemoteSparkWCompare.java | 73 +
.../parfor/ResultMergeTaggedMatrixIndexes.java | 116 +
.../runtime/controlprogram/parfor/Task.java | 235 +
.../controlprogram/parfor/TaskPartitioner.java | 84 +
.../parfor/TaskPartitionerFactoring.java | 208 +
.../parfor/TaskPartitionerFactoringCmax.java | 50 +
.../parfor/TaskPartitionerFactoringCmin.java | 49 +
.../parfor/TaskPartitionerFixedsize.java | 158 +
.../parfor/TaskPartitionerNaive.java | 38 +
.../parfor/TaskPartitionerStatic.java | 38 +
.../parfor/mqo/MergedMRJobInstruction.java | 76 +
.../parfor/mqo/PiggybackingWorker.java | 153 +
.../mqo/PiggybackingWorkerTimeSequential.java | 101 +
.../PiggybackingWorkerUtilDecayParallel.java | 182 +
.../mqo/PiggybackingWorkerUtilTimeParallel.java | 169 +
.../parfor/mqo/RuntimePiggybacking.java | 252 +
.../parfor/mqo/RuntimePiggybackingUtils.java | 52 +
.../parfor/opt/CostEstimator.java | 330 +
.../parfor/opt/CostEstimatorHops.java | 140 +
.../parfor/opt/CostEstimatorRuntime.java | 275 +
.../controlprogram/parfor/opt/CostFunction.java | 172 +
.../controlprogram/parfor/opt/MemoTable.java | 160 +
.../parfor/opt/MemoTableEntry.java | 112 +
.../controlprogram/parfor/opt/OptNode.java | 719 ++
.../parfor/opt/OptNodeStatistics.java | 128 +
.../controlprogram/parfor/opt/OptTree.java | 111 +
.../parfor/opt/OptTreeConverter.java | 1104 +++
.../parfor/opt/OptTreePlanChecker.java | 257 +
.../parfor/opt/OptTreePlanMapping.java | 75 +
.../parfor/opt/OptTreePlanMappingAbstract.java | 122 +
.../parfor/opt/OptTreePlanMappingRuntime.java | 86 +
.../parfor/opt/OptimizationWrapper.java | 528 ++
.../controlprogram/parfor/opt/Optimizer.java | 175 +
.../parfor/opt/OptimizerConstrained.java | 441 ++
.../parfor/opt/OptimizerHeuristic.java | 80 +
.../parfor/opt/OptimizerRuleBased.java | 3182 +++++++++
.../controlprogram/parfor/opt/PerfTestDef.java | 98 +
.../parfor/opt/PerfTestExtFunctCP.java | 92 +
.../parfor/opt/PerfTestMemoryObserver.java | 120 +
.../controlprogram/parfor/opt/PerfTestTool.java | 1751 +++++
.../parfor/opt/PerfTestToolRegression.dml | 55 +
.../parfor/opt/ProgramRecompiler.java | 721 ++
.../parfor/stat/InfrastructureAnalyzer.java | 551 ++
.../controlprogram/parfor/stat/Stat.java | 60 +
.../parfor/stat/StatisticMonitor.java | 245 +
.../controlprogram/parfor/stat/Timing.java | 81 +
.../controlprogram/parfor/util/Cell.java | 70 +
.../controlprogram/parfor/util/IDHandler.java | 186 +
.../controlprogram/parfor/util/IDSequence.java | 79 +
.../parfor/util/PairWritableBlock.java | 59 +
.../parfor/util/PairWritableCell.java | 59 +
.../parfor/util/StagingFileUtils.java | 257 +
.../sysml/runtime/functionobjects/And.java | 49 +
.../sysml/runtime/functionobjects/Builtin.java | 528 ++
.../sysml/runtime/functionobjects/CM.java | 467 ++
.../sysml/runtime/functionobjects/COV.java | 147 +
.../sysml/runtime/functionobjects/CTable.java | 184 +
.../runtime/functionobjects/DiagIndex.java | 78 +
.../sysml/runtime/functionobjects/Divide.java | 62 +
.../sysml/runtime/functionobjects/Equals.java | 90 +
.../runtime/functionobjects/FileFunction.java | 23 +
.../runtime/functionobjects/FunctionObject.java | 149 +
.../runtime/functionobjects/GreaterThan.java | 85 +
.../functionobjects/GreaterThanEquals.java | 85 +
.../runtime/functionobjects/IndexFunction.java | 27 +
.../runtime/functionobjects/IntegerDivide.java | 86 +
.../runtime/functionobjects/KahanFunction.java | 43 +
.../runtime/functionobjects/KahanPlus.java | 110 +
.../runtime/functionobjects/KahanPlusSq.java | 101 +
.../sysml/runtime/functionobjects/LessThan.java | 85 +
.../runtime/functionobjects/LessThanEquals.java | 85 +
.../sysml/runtime/functionobjects/MaxIndex.java | 75 +
.../sysml/runtime/functionobjects/Mean.java | 69 +
.../sysml/runtime/functionobjects/MinIndex.java | 75 +
.../sysml/runtime/functionobjects/Minus.java | 69 +
.../runtime/functionobjects/Minus1Multiply.java | 68 +
.../sysml/runtime/functionobjects/MinusNz.java | 58 +
.../sysml/runtime/functionobjects/Modulus.java | 79 +
.../sysml/runtime/functionobjects/Multiply.java | 69 +
.../runtime/functionobjects/Multiply2.java | 71 +
.../sysml/runtime/functionobjects/Not.java | 46 +
.../runtime/functionobjects/NotEquals.java | 89 +
.../functionobjects/OffsetColumnIndex.java | 81 +
.../sysml/runtime/functionobjects/Or.java | 49 +
.../functionobjects/ParameterizedBuiltin.java | 310 +
.../sysml/runtime/functionobjects/Plus.java | 77 +
.../sysml/runtime/functionobjects/Power.java | 64 +
.../sysml/runtime/functionobjects/Power2.java | 74 +
.../runtime/functionobjects/ReduceAll.java | 73 +
.../runtime/functionobjects/ReduceCol.java | 73 +
.../runtime/functionobjects/ReduceDiag.java | 73 +
.../runtime/functionobjects/ReduceRow.java | 75 +
.../runtime/functionobjects/RemoveFile.java | 56 +
.../runtime/functionobjects/RenameFile.java | 56 +
.../runtime/functionobjects/SortIndex.java | 77 +
.../runtime/functionobjects/SwapIndex.java | 73 +
.../runtime/functionobjects/ValueFunction.java | 25 +
.../instructions/CPInstructionParser.java | 402 ++
.../sysml/runtime/instructions/Instruction.java | 233 +
.../runtime/instructions/InstructionParser.java | 79 +
.../runtime/instructions/InstructionUtils.java | 809 +++
.../instructions/MRInstructionParser.java | 619 ++
.../runtime/instructions/MRJobInstruction.java | 1521 +++++
.../instructions/SPInstructionParser.java | 416 ++
.../cp/AggregateBinaryCPInstruction.java | 89 +
.../cp/AggregateTernaryCPInstruction.java | 91 +
.../cp/AggregateUnaryCPInstruction.java | 159 +
.../instructions/cp/AppendCPInstruction.java | 140 +
.../cp/ArithmeticBinaryCPInstruction.java | 112 +
.../instructions/cp/BinaryCPInstruction.java | 67 +
.../cp/BooleanBinaryCPInstruction.java | 71 +
.../runtime/instructions/cp/BooleanObject.java | 73 +
.../cp/BooleanUnaryCPInstruction.java | 69 +
.../instructions/cp/BreakPointInstruction.java | 122 +
.../cp/BuiltinBinaryCPInstruction.java | 67 +
.../cp/BuiltinUnaryCPInstruction.java | 81 +
.../runtime/instructions/cp/CM_COV_Object.java | 204 +
.../runtime/instructions/cp/CPInstruction.java | 92 +
.../runtime/instructions/cp/CPOperand.java | 116 +
.../cp/CentralMomentCPInstruction.java | 132 +
.../cp/ComputationCPInstruction.java | 75 +
.../cp/CovarianceCPInstruction.java | 120 +
.../sysml/runtime/instructions/cp/Data.java | 80 +
.../instructions/cp/DataGenCPInstruction.java | 324 +
.../cp/DataPartitionCPInstruction.java | 112 +
.../runtime/instructions/cp/DoubleObject.java | 73 +
.../instructions/cp/FileCPInstruction.java | 130 +
.../runtime/instructions/cp/FileObject.java | 69 +
.../cp/FunctionCallCPInstruction.java | 294 +
.../runtime/instructions/cp/IntObject.java | 75 +
.../runtime/instructions/cp/KahanObject.java | 101 +
.../instructions/cp/MMChainCPInstruction.java | 107 +
.../instructions/cp/MMTSJCPInstruction.java | 90 +
.../cp/MatrixBuiltinCPInstruction.java | 62 +
.../cp/MatrixIndexingCPInstruction.java | 197 +
.../cp/MatrixMatrixArithmeticCPInstruction.java | 64 +
.../cp/MatrixMatrixBuiltinCPInstruction.java | 73 +
.../cp/MatrixMatrixRelationalCPInstruction.java | 62 +
.../cp/MatrixReshapeCPInstruction.java | 95 +
.../cp/MatrixScalarBuiltinCPInstruction.java | 65 +
.../cp/MultiReturnBuiltinCPInstruction.java | 116 +
.../instructions/cp/PMMJCPInstruction.java | 86 +
.../cp/ParameterizedBuiltinCPInstruction.java | 245 +
.../cp/QuantilePickCPInstruction.java | 222 +
.../cp/QuantileSortCPInstruction.java | 106 +
.../cp/QuaternaryCPInstruction.java | 145 +
.../cp/RelationalBinaryCPInstruction.java | 99 +
.../instructions/cp/ReorgCPInstruction.java | 135 +
.../cp/ScalarBuiltinCPInstruction.java | 82 +
.../cp/ScalarMatrixArithmeticCPInstruction.java | 65 +
.../cp/ScalarMatrixRelationalCPInstruction.java | 65 +
.../runtime/instructions/cp/ScalarObject.java | 49 +
.../cp/ScalarScalarArithmeticCPInstruction.java | 89 +
.../cp/ScalarScalarBuiltinCPInstruction.java | 92 +
.../cp/ScalarScalarRelationalCPInstruction.java | 76 +
.../cp/StringInitCPInstruction.java | 117 +
.../runtime/instructions/cp/StringObject.java | 91 +
.../instructions/cp/TernaryCPInstruction.java | 182 +
.../cp/UaggOuterChainCPInstruction.java | 129 +
.../instructions/cp/UnaryCPInstruction.java | 99 +
.../instructions/cp/VariableCPInstruction.java | 1021 +++
.../cpfile/MatrixIndexingCPFileInstruction.java | 153 +
.../ParameterizedBuiltinCPFileInstruction.java | 1168 ++++
.../mr/AggregateBinaryInstruction.java | 270 +
.../instructions/mr/AggregateInstruction.java | 75 +
.../mr/AggregateUnaryInstruction.java | 116 +
.../instructions/mr/AppendGInstruction.java | 191 +
.../instructions/mr/AppendInstruction.java | 83 +
.../instructions/mr/AppendMInstruction.java | 153 +
.../instructions/mr/AppendRInstruction.java | 74 +
.../mr/BinUaggChainInstruction.java | 124 +
.../instructions/mr/BinaryInstruction.java | 112 +
.../instructions/mr/BinaryMInstruction.java | 126 +
.../mr/BinaryMRInstructionBase.java | 44 +
.../instructions/mr/CM_N_COVInstruction.java | 93 +
.../instructions/mr/CSVReblockInstruction.java | 80 +
.../instructions/mr/CSVWriteInstruction.java | 69 +
.../mr/CombineBinaryInstruction.java | 77 +
.../mr/CombineTernaryInstruction.java | 66 +
.../mr/CombineUnaryInstruction.java | 68 +
.../mr/CumulativeAggregateInstruction.java | 106 +
.../mr/CumulativeOffsetInstruction.java | 107 +
.../mr/CumulativeSplitInstruction.java | 112 +
.../instructions/mr/DataGenMRInstruction.java | 83 +
.../mr/DataPartitionMRInstruction.java | 50 +
.../mr/GroupedAggregateInstruction.java | 99 +
.../mr/IDistributedCacheConsumer.java | 45 +
.../instructions/mr/MMTSJMRInstruction.java | 118 +
.../runtime/instructions/mr/MRInstruction.java | 98 +
.../mr/MapMultChainInstruction.java | 267 +
.../mr/MatrixReshapeMRInstruction.java | 131 +
.../instructions/mr/PMMJMRInstruction.java | 156 +
.../mr/ParameterizedBuiltinMRInstruction.java | 181 +
.../instructions/mr/PickByCountInstruction.java | 124 +
.../instructions/mr/QuaternaryInstruction.java | 348 +
.../instructions/mr/RandInstruction.java | 110 +
.../mr/RangeBasedReIndexInstruction.java | 130 +
.../instructions/mr/ReblockInstruction.java | 73 +
.../mr/RemoveEmptyMRInstruction.java | 109 +
.../instructions/mr/ReorgInstruction.java | 162 +
.../instructions/mr/ReplicateInstruction.java | 163 +
.../instructions/mr/ScalarInstruction.java | 115 +
.../runtime/instructions/mr/SeqInstruction.java | 78 +
.../instructions/mr/TernaryInstruction.java | 317 +
.../mr/UaggOuterChainInstruction.java | 246 +
.../instructions/mr/UnaryInstruction.java | 96 +
.../instructions/mr/UnaryMRInstructionBase.java | 43 +
.../instructions/mr/ZeroOutInstruction.java | 116 +
.../spark/AggregateTernarySPInstruction.java | 175 +
.../spark/AggregateUnarySPInstruction.java | 251 +
.../spark/AppendGAlignedSPInstruction.java | 120 +
.../spark/AppendGSPInstruction.java | 251 +
.../spark/AppendMSPInstruction.java | 290 +
.../spark/AppendRSPInstruction.java | 116 +
.../spark/ArithmeticBinarySPInstruction.java | 94 +
.../spark/BinUaggChainSPInstruction.java | 122 +
.../instructions/spark/BinarySPInstruction.java | 363 +
.../spark/BuiltinBinarySPInstruction.java | 81 +
.../spark/BuiltinUnarySPInstruction.java | 41 +
.../spark/CSVReblockSPInstruction.java | 132 +
.../spark/CentralMomentSPInstruction.java | 221 +
.../spark/CheckpointSPInstruction.java | 158 +
.../spark/ComputationSPInstruction.java | 105 +
.../spark/CovarianceSPInstruction.java | 208 +
.../instructions/spark/CpmmSPInstruction.java | 191 +
.../spark/CumulativeAggregateSPInstruction.java | 147 +
.../spark/CumulativeOffsetSPInstruction.java | 220 +
.../spark/MapmmChainSPInstruction.java | 219 +
.../instructions/spark/MapmmSPInstruction.java | 384 ++
.../MatrixBVectorArithmeticSPInstruction.java | 55 +
.../MatrixBVectorBuiltinSPInstruction.java | 54 +
.../MatrixBVectorRelationalSPInstruction.java | 55 +
.../spark/MatrixBuiltinSPInstruction.java | 82 +
.../spark/MatrixIndexingSPInstruction.java | 526 ++
.../MatrixMatrixArithmeticSPInstruction.java | 51 +
.../spark/MatrixMatrixBuiltinSPInstruction.java | 41 +
.../MatrixMatrixRelationalSPInstruction.java | 50 +
.../spark/MatrixReshapeSPInstruction.java | 154 +
.../MatrixScalarArithmeticSPInstruction.java | 50 +
.../spark/MatrixScalarBuiltinSPInstruction.java | 48 +
.../MatrixScalarRelationalSPInstruction.java | 31 +
.../instructions/spark/PMapmmSPInstruction.java | 223 +
.../ParameterizedBuiltinSPInstruction.java | 587 ++
.../instructions/spark/PmmSPInstruction.java | 189 +
.../spark/QuantilePickSPInstruction.java | 293 +
.../spark/QuantileSortSPInstruction.java | 123 +
.../spark/QuaternarySPInstruction.java | 571 ++
.../instructions/spark/RandSPInstruction.java | 753 +++
.../spark/ReblockSPInstruction.java | 185 +
.../spark/RelationalBinarySPInstruction.java | 67 +
.../instructions/spark/ReorgSPInstruction.java | 284 +
.../instructions/spark/RmmSPInstruction.java | 205 +
.../instructions/spark/SPInstruction.java | 139 +
.../spark/TernarySPInstruction.java | 523 ++
.../instructions/spark/TsmmSPInstruction.java | 118 +
.../spark/UaggOuterChainSPInstruction.java | 400 ++
.../instructions/spark/UnarySPInstruction.java | 99 +
.../instructions/spark/WriteSPInstruction.java | 290 +
.../instructions/spark/ZipmmSPInstruction.java | 148 +
.../spark/data/BlockPartitioner.java | 114 +
.../spark/data/BroadcastObject.java | 53 +
.../spark/data/CorrMatrixBlock.java | 139 +
.../instructions/spark/data/CountLinesInfo.java | 38 +
.../spark/data/LazyIterableIterator.java | 88 +
.../instructions/spark/data/LineageObject.java | 81 +
.../spark/data/PartitionedBroadcastMatrix.java | 121 +
.../spark/data/PartitionedMatrixBlock.java | 385 ++
.../instructions/spark/data/RDDObject.java | 122 +
.../instructions/spark/data/RDDProperties.java | 55 +
.../instructions/spark/data/RowMatrixBlock.java | 135 +
.../spark/data/SerLongWritable.java | 36 +
.../instructions/spark/data/SerText.java | 36 +
.../AggregateDropCorrectionFunction.java | 50 +
.../ComputeBinaryBlockNnzFunction.java | 45 +
.../ConvertColumnRDDToBinaryBlock.java | 68 +
.../ConvertMLLibBlocksToBinaryBlocks.java | 77 +
.../functions/ConvertMatrixBlockToIJVLines.java | 67 +
.../functions/ConvertStringToLongTextPair.java | 34 +
.../ConvertTextLineToBinaryCellFunction.java | 91 +
.../spark/functions/ConvertTextToString.java | 31 +
.../spark/functions/CopyBinaryCellFunction.java | 43 +
.../spark/functions/CopyBlockFunction.java | 50 +
.../spark/functions/CopyBlockPairFunction.java | 60 +
.../spark/functions/CopyTextInputFunction.java | 41 +
.../functions/ExtractBlockForBinaryReblock.java | 129 +
.../spark/functions/ExtractGroup.java | 91 +
.../spark/functions/ExtractGroupNWeights.java | 120 +
.../functions/FilterDiagBlocksFunction.java | 45 +
.../functions/FilterNonEmptyBlocksFunction.java | 48 +
.../FindMatrixBlockFromMatrixIndexes.java | 44 +
.../spark/functions/GetMIMBFromRow.java | 44 +
.../spark/functions/GetMLBlock.java | 40 +
.../spark/functions/GetMLLibBlocks.java | 121 +
.../spark/functions/IsBlockInRange.java | 52 +
.../spark/functions/LastCellInMatrixBlock.java | 53 +
.../functions/MatrixMatrixBinaryOpFunction.java | 44 +
.../functions/MatrixScalarUnaryFunction.java | 45 +
.../functions/MatrixVectorBinaryOpFunction.java | 63 +
.../MatrixVectorBinaryOpPartitionFunction.java | 84 +
.../functions/OuterVectorBinaryOpFunction.java | 104 +
.../functions/PerformGroupByAggInCombiner.java | 101 +
.../functions/PerformGroupByAggInReducer.java | 110 +
.../spark/functions/ReorgMapFunction.java | 74 +
.../functions/ReplicateVectorFunction.java | 69 +
.../spark/functions/SparkListener.java | 176 +
.../UnflattenIterablesAfterCogroup.java | 39 +
.../spark/utils/RDDAggregateUtils.java | 777 +++
.../spark/utils/RDDConverterUtils.java | 748 +++
.../spark/utils/RDDConverterUtilsExt.java | 715 ++
.../instructions/spark/utils/RDDSortUtils.java | 843 +++
.../instructions/spark/utils/SparkUtils.java | 448 ++
.../runtime/io/BinaryBlockSerialization.java | 142 +
.../sysml/runtime/io/IOUtilFunctions.java | 203 +
.../apache/sysml/runtime/io/MatrixReader.java | 137 +
.../sysml/runtime/io/MatrixReaderFactory.java | 115 +
.../apache/sysml/runtime/io/MatrixWriter.java | 142 +
.../sysml/runtime/io/MatrixWriterFactory.java | 98 +
.../apache/sysml/runtime/io/ReadProperties.java | 46 +
.../sysml/runtime/io/ReaderBinaryBlock.java | 259 +
.../runtime/io/ReaderBinaryBlockParallel.java | 223 +
.../sysml/runtime/io/ReaderBinaryCell.java | 140 +
.../apache/sysml/runtime/io/ReaderTextCSV.java | 259 +
.../sysml/runtime/io/ReaderTextCSVParallel.java | 533 ++
.../apache/sysml/runtime/io/ReaderTextCell.java | 299 +
.../runtime/io/ReaderTextCellParallel.java | 357 +
.../sysml/runtime/io/WriterBinaryBlock.java | 417 ++
.../runtime/io/WriterBinaryBlockParallel.java | 202 +
.../sysml/runtime/io/WriterBinaryCell.java | 147 +
.../sysml/runtime/io/WriterMatrixMarket.java | 236 +
.../runtime/io/WriterMatrixMarketParallel.java | 207 +
.../apache/sysml/runtime/io/WriterTextCSV.java | 442 ++
.../sysml/runtime/io/WriterTextCSVParallel.java | 274 +
.../apache/sysml/runtime/io/WriterTextCell.java | 158 +
.../runtime/io/WriterTextCellParallel.java | 195 +
.../apache/sysml/runtime/matrix/CMCOVMR.java | 134 +
.../sysml/runtime/matrix/CSVReblockMR.java | 473 ++
.../apache/sysml/runtime/matrix/CleanupMR.java | 180 +
.../apache/sysml/runtime/matrix/CombineMR.java | 377 ++
.../apache/sysml/runtime/matrix/DataGenMR.java | 397 ++
.../sysml/runtime/matrix/DataPartitionMR.java | 100 +
.../org/apache/sysml/runtime/matrix/GMR.java | 424 ++
.../sysml/runtime/matrix/GroupedAggMR.java | 144 +
.../apache/sysml/runtime/matrix/JobReturn.java | 90 +
.../org/apache/sysml/runtime/matrix/MMCJMR.java | 303 +
.../org/apache/sysml/runtime/matrix/MMRJMR.java | 199 +
.../runtime/matrix/MatrixCharacteristics.java | 423 ++
.../matrix/MatrixDimensionsMetaData.java | 76 +
.../runtime/matrix/MatrixFormatMetaData.java | 51 +
.../apache/sysml/runtime/matrix/MetaData.java | 44 +
.../apache/sysml/runtime/matrix/ReblockMR.java | 228 +
.../org/apache/sysml/runtime/matrix/SortMR.java | 414 ++
.../apache/sysml/runtime/matrix/WriteCSVMR.java | 190 +
.../matrix/data/AdaptivePartialBlock.java | 160 +
.../matrix/data/AddDummyWeightConverter.java | 81 +
.../data/BinaryBlockToBinaryCellConverter.java | 137 +
.../data/BinaryBlockToRowBlockConverter.java | 108 +
.../data/BinaryBlockToTextCellConverter.java | 142 +
.../data/BinaryCellToRowBlockConverter.java | 57 +
.../matrix/data/BinaryCellToTextConverter.java | 57 +
.../sysml/runtime/matrix/data/CM_N_COVCell.java | 382 ++
.../matrix/data/CSVFileFormatProperties.java | 110 +
.../sysml/runtime/matrix/data/CTableMap.java | 139 +
.../sysml/runtime/matrix/data/Converter.java | 30 +
.../matrix/data/FileFormatProperties.java | 43 +
.../apache/sysml/runtime/matrix/data/IJV.java | 52 +
.../runtime/matrix/data/IdenticalConverter.java | 51 +
.../sysml/runtime/matrix/data/InputInfo.java | 159 +
.../runtime/matrix/data/LibCommonsMath.java | 244 +
.../sysml/runtime/matrix/data/LibMatrixAgg.java | 2863 ++++++++
.../runtime/matrix/data/LibMatrixBincell.java | 1320 ++++
.../runtime/matrix/data/LibMatrixDatagen.java | 806 +++
.../runtime/matrix/data/LibMatrixMult.java | 4285 ++++++++++++
.../runtime/matrix/data/LibMatrixOuterAgg.java | 1599 +++++
.../runtime/matrix/data/LibMatrixReorg.java | 2116 ++++++
.../sysml/runtime/matrix/data/MatrixBlock.java | 6295 ++++++++++++++++++
.../matrix/data/MatrixBlockDataInput.java | 57 +
.../matrix/data/MatrixBlockDataOutput.java | 55 +
.../sysml/runtime/matrix/data/MatrixCell.java | 491 ++
.../runtime/matrix/data/MatrixIndexes.java | 198 +
.../runtime/matrix/data/MatrixPackedCell.java | 197 +
.../sysml/runtime/matrix/data/MatrixValue.java | 212 +
.../matrix/data/MultipleOutputCommitter.java | 151 +
.../data/NumItemsByEachReducerMetaData.java | 67 +
.../matrix/data/OperationsOnMatrixValues.java | 351 +
.../sysml/runtime/matrix/data/OutputInfo.java | 150 +
.../apache/sysml/runtime/matrix/data/Pair.java | 63 +
.../sysml/runtime/matrix/data/PartialBlock.java | 103 +
.../data/PoissonRandomMatrixGenerator.java | 49 +
.../matrix/data/RandomMatrixGenerator.java | 101 +
.../runtime/matrix/data/RuntimeDataFormat.java | 61 +
.../sysml/runtime/matrix/data/SparseRow.java | 524 ++
.../runtime/matrix/data/SparseRowsIterator.java | 108 +
.../sysml/runtime/matrix/data/Tagged.java | 120 +
.../matrix/data/TaggedAdaptivePartialBlock.java | 34 +
.../matrix/data/TaggedFirstSecondIndexes.java | 216 +
.../sysml/runtime/matrix/data/TaggedInt.java | 59 +
.../runtime/matrix/data/TaggedMatrixBlock.java | 39 +
.../runtime/matrix/data/TaggedMatrixCell.java | 39 +
.../matrix/data/TaggedMatrixPackedCell.java | 39 +
.../runtime/matrix/data/TaggedMatrixValue.java | 48 +
.../runtime/matrix/data/TaggedPartialBlock.java | 33 +
.../matrix/data/TaggedTripleIndexes.java | 154 +
.../data/TextCellToRowBlockConverter.java | 83 +
.../matrix/data/TextToBinaryCellConverter.java | 84 +
.../runtime/matrix/data/TripleIndexes.java | 164 +
.../matrix/data/UnPaddedOutputFormat.java | 64 +
.../sysml/runtime/matrix/data/WeightedCell.java | 191 +
.../data/WeightedCellToSortInputConverter.java | 58 +
.../sysml/runtime/matrix/data/WeightedPair.java | 110 +
.../data/WeightedPairToSortInputConverter.java | 58 +
.../data/hadoopfix/DelegatingInputFormat.java | 137 +
.../matrix/data/hadoopfix/MultipleInputs.java | 133 +
.../matrix/data/hadoopfix/TaggedInputSplit.java | 141 +
.../runtime/matrix/mapred/CMCOVMRMapper.java | 161 +
.../runtime/matrix/mapred/CMCOVMRReducer.java | 153 +
.../matrix/mapred/CSVAssignRowIDMapper.java | 157 +
.../matrix/mapred/CSVAssignRowIDReducer.java | 63 +
.../runtime/matrix/mapred/CSVReblockMapper.java | 200 +
.../matrix/mapred/CSVReblockReducer.java | 124 +
.../runtime/matrix/mapred/CSVWriteMapper.java | 108 +
.../runtime/matrix/mapred/CSVWriteReducer.java | 384 ++
.../sysml/runtime/matrix/mapred/CachedMap.java | 133 +
.../runtime/matrix/mapred/CachedMapElement.java | 27 +
.../runtime/matrix/mapred/CachedValueMap.java | 150 +
.../mapred/CollectMultipleConvertedOutputs.java | 76 +
.../runtime/matrix/mapred/DataGenMapper.java | 136 +
.../matrix/mapred/DistributedCacheInput.java | 227 +
.../runtime/matrix/mapred/GMRCombiner.java | 66 +
.../runtime/matrix/mapred/GMRCtableBuffer.java | 237 +
.../sysml/runtime/matrix/mapred/GMRMapper.java | 271 +
.../sysml/runtime/matrix/mapred/GMRReducer.java | 173 +
.../matrix/mapred/GroupedAggMRCombiner.java | 167 +
.../matrix/mapred/GroupedAggMRMapper.java | 123 +
.../matrix/mapred/GroupedAggMRReducer.java | 153 +
.../matrix/mapred/IndexedMatrixValue.java | 85 +
.../runtime/matrix/mapred/MMCJMRCache.java | 161 +
.../runtime/matrix/mapred/MMCJMRCombiner.java | 53 +
.../mapred/MMCJMRCombinerReducerBase.java | 143 +
.../runtime/matrix/mapred/MMCJMRInputCache.java | 160 +
.../runtime/matrix/mapred/MMCJMRMapper.java | 132 +
.../runtime/matrix/mapred/MMCJMRReducer.java | 303 +
.../mapred/MMCJMRReducerWithAggregator.java | 251 +
.../runtime/matrix/mapred/MMRJMRMapper.java | 139 +
.../runtime/matrix/mapred/MMRJMRReducer.java | 165 +
.../mapred/MRBaseForCommonInstructions.java | 337 +
.../matrix/mapred/MRConfigurationNames.java | 68 +
.../matrix/mapred/MRJobConfiguration.java | 1909 ++++++
.../sysml/runtime/matrix/mapred/MapperBase.java | 433 ++
.../matrix/mapred/PartialAggregator.java | 258 +
.../runtime/matrix/mapred/ReblockBuffer.java | 408 ++
.../runtime/matrix/mapred/ReblockMapper.java | 228 +
.../runtime/matrix/mapred/ReblockReducer.java | 156 +
.../sysml/runtime/matrix/mapred/ReduceBase.java | 450 ++
.../operators/AggregateBinaryOperator.java | 60 +
.../matrix/operators/AggregateOperator.java | 63 +
.../operators/AggregateUnaryOperator.java | 70 +
.../matrix/operators/BinaryOperator.java | 101 +
.../runtime/matrix/operators/CMOperator.java | 120 +
.../runtime/matrix/operators/COVOperator.java | 36 +
.../matrix/operators/LeftScalarOperator.java | 66 +
.../runtime/matrix/operators/Operator.java | 39 +
.../matrix/operators/QuaternaryOperator.java | 97 +
.../matrix/operators/ReIndexOperator.java | 30 +
.../runtime/matrix/operators/ReorgOperator.java | 39 +
.../matrix/operators/RightScalarOperator.java | 58 +
.../matrix/operators/ScalarOperator.java | 99 +
.../matrix/operators/SimpleOperator.java | 36 +
.../runtime/matrix/operators/UnaryOperator.java | 47 +
.../matrix/operators/ZeroOutOperator.java | 30 +
.../sort/CompactDoubleIntInputFormat.java | 211 +
.../runtime/matrix/sort/CompactInputFormat.java | 122 +
.../matrix/sort/CompactOutputFormat.java | 90 +
.../matrix/sort/IndexSortComparable.java | 82 +
.../matrix/sort/IndexSortComparableDesc.java | 47 +
.../runtime/matrix/sort/IndexSortMapper.java | 73 +
.../runtime/matrix/sort/IndexSortReducer.java | 101 +
.../matrix/sort/IndexSortStitchupMapper.java | 139 +
.../matrix/sort/IndexSortStitchupReducer.java | 72 +
.../matrix/sort/PickFromCompactInputFormat.java | 545 ++
.../runtime/matrix/sort/ReadWithZeros.java | 82 +
.../matrix/sort/SamplingSortMRInputFormat.java | 237 +
.../runtime/matrix/sort/ValueSortMapper.java | 95 +
.../runtime/matrix/sort/ValueSortReducer.java | 66 +
.../sysml/runtime/transform/ApplyTfBBMR.java | 155 +
.../runtime/transform/ApplyTfBBMapper.java | 149 +
.../sysml/runtime/transform/ApplyTfCSVMR.java | 126 +
.../runtime/transform/ApplyTfCSVMapper.java | 110 +
.../runtime/transform/ApplyTfCSVSPARK.java | 158 +
.../sysml/runtime/transform/BinAgent.java | 353 +
.../sysml/runtime/transform/DataTransform.java | 1496 +++++
.../sysml/runtime/transform/DistinctValue.java | 106 +
.../sysml/runtime/transform/DummycodeAgent.java | 424 ++
.../sysml/runtime/transform/GTFMTDMapper.java | 105 +
.../sysml/runtime/transform/GTFMTDReducer.java | 122 +
.../sysml/runtime/transform/GenTfMtdMR.java | 104 +
.../sysml/runtime/transform/GenTfMtdSPARK.java | 233 +
.../sysml/runtime/transform/MVImputeAgent.java | 975 +++
.../sysml/runtime/transform/OmitAgent.java | 122 +
.../sysml/runtime/transform/RecodeAgent.java | 463 ++
.../apache/sysml/runtime/transform/TfUtils.java | 547 ++
.../runtime/transform/TransformationAgent.java | 91 +
.../runtime/util/BinaryBlockInputFormat.java | 45 +
.../runtime/util/BinaryBlockRecordReader.java | 67 +
.../sysml/runtime/util/DataConverter.java | 691 ++
.../util/FastBufferedDataInputStream.java | 288 +
.../util/FastBufferedDataOutputStream.java | 325 +
.../sysml/runtime/util/FastStringTokenizer.java | 162 +
.../apache/sysml/runtime/util/IndexRange.java | 48 +
.../sysml/runtime/util/LocalFileUtils.java | 558 ++
.../runtime/util/LongLongDoubleHashMap.java | 169 +
.../sysml/runtime/util/MapReduceTool.java | 725 ++
.../sysml/runtime/util/NormalPRNGenerator.java | 72 +
.../apache/sysml/runtime/util/PRNGenerator.java | 38 +
.../sysml/runtime/util/PoissonPRNGenerator.java | 82 +
.../org/apache/sysml/runtime/util/RandN.java | 65 +
.../apache/sysml/runtime/util/RandNPair.java | 51 +
.../apache/sysml/runtime/util/SortUtils.java | 370 +
.../sysml/runtime/util/UniformPRNGenerator.java | 49 +
.../sysml/runtime/util/UtilFunctions.java | 333 +
.../java/org/apache/sysml/udf/BinaryObject.java | 50 +
.../ExternalFunctionInvocationInstruction.java | 81 +
.../org/apache/sysml/udf/FunctionParameter.java | 62 +
src/main/java/org/apache/sysml/udf/Matrix.java | 233 +
.../org/apache/sysml/udf/PackageFunction.java | 184 +
.../sysml/udf/PackageRuntimeException.java | 43 +
src/main/java/org/apache/sysml/udf/Scalar.java | 69 +
.../apache/sysml/udf/lib/BinningWrapper.java | 109 +
.../org/apache/sysml/udf/lib/DeNaNWrapper.java | 77 +
.../sysml/udf/lib/DeNegInfinityWrapper.java | 77 +
.../sysml/udf/lib/DynamicProjectMatrixCP.java | 103 +
.../sysml/udf/lib/DynamicReadMatrixCP.java | 84 +
.../sysml/udf/lib/DynamicReadMatrixRcCP.java | 95 +
.../sysml/udf/lib/DynamicWriteMatrixCP.java | 82 +
.../org/apache/sysml/udf/lib/GatherWrapper.java | 96 +
.../org/apache/sysml/udf/lib/OrderWrapper.java | 123 +
.../sysml/udf/lib/PermutationMatrixWrapper.java | 153 +
.../apache/sysml/udf/lib/RemoveEmptyRows.java | 153 +
.../org/apache/sysml/udf/lib/TimeWrapper.java | 74 +
.../org/apache/sysml/utils/AppException.java | 38 +
.../java/org/apache/sysml/utils/Explain.java | 1155 ++++
.../InstallDependencyForIntegrationTests.java | 145 +
.../java/org/apache/sysml/utils/JSONHelper.java | 53 +
.../apache/sysml/utils/ParameterBuilder.java | 110 +
.../java/org/apache/sysml/utils/Statistics.java | 549 ++
src/main/java/org/apache/sysml/utils/Timer.java | 60 +
.../org/apache/sysml/yarn/DMLAppMaster.java | 177 +
.../sysml/yarn/DMLAppMasterStatusReporter.java | 68 +
.../apache/sysml/yarn/DMLAppMasterUtils.java | 223 +
.../org/apache/sysml/yarn/DMLYarnClient.java | 588 ++
.../apache/sysml/yarn/DMLYarnClientProxy.java | 114 +
.../apache/sysml/yarn/ropt/GridEnumeration.java | 52 +
.../sysml/yarn/ropt/GridEnumerationEqui.java | 61 +
.../sysml/yarn/ropt/GridEnumerationExp.java | 62 +
.../sysml/yarn/ropt/GridEnumerationHybrid.java | 61 +
.../sysml/yarn/ropt/GridEnumerationMemory.java | 178 +
.../yarn/ropt/MRJobResourceInstruction.java | 50 +
.../apache/sysml/yarn/ropt/ResourceConfig.java | 214 +
.../sysml/yarn/ropt/ResourceOptimizer.java | 800 +++
.../yarn/ropt/ResourceOptimizerCPMigration.java | 265 +
.../sysml/yarn/ropt/YarnClusterAnalyzer.java | 742 +++
.../sysml/yarn/ropt/YarnClusterConfig.java | 80 +
.../sysml/yarn/ropt/YarnOptimizerUtils.java | 112 +
.../AutomatedScalabilityTestBase.java | 91 -
.../dml/test/integration/AutomatedTestBase.java | 1627 -----
.../BinaryMatrixCharacteristics.java | 84 -
.../dml/test/integration/TestConfiguration.java | 174 -
.../integration/TestMatrixCharacteristics.java | 100 -
.../applications/ApplyTransformTest.java | 176 -
.../integration/applications/ArimaTest.java | 119 -
.../integration/applications/CsplineCGTest.java | 103 -
.../integration/applications/CsplineDSTest.java | 110 -
.../test/integration/applications/GLMTest.java | 697 --
.../test/integration/applications/GNMFTest.java | 133 -
.../test/integration/applications/HITSTest.java | 88 -
.../test/integration/applications/ID3Test.java | 122 -
.../integration/applications/L2SVMTest.java | 116 -
.../applications/LinearLogRegTest.java | 114 -
.../applications/LinearRegressionTest.java | 109 -
.../applications/MDABivariateStatsTest.java | 133 -
.../applications/MultiClassSVMTest.java | 138 -
.../applications/NaiveBayesTest.java | 122 -
.../integration/applications/PageRankTest.java | 112 -
.../integration/applications/WelchTTest.java | 104 -
.../BivariateCategoricalCategoricallTest.java | 256 -
.../BivariateOrdinalOrdinalTest.java | 161 -
.../BivariateScaleCategoricalTest.java | 181 -
.../BivariateScaleScaleTest.java | 177 -
.../descriptivestats/OrderStatisticsTest.java | 158 -
.../UnivariateCategoricalTest.java | 145 -
.../descriptivestats/UnivariateStatsBase.java | 294 -
.../UnivariateUnweightedScaleDenseTest.java | 164 -
.../UnivariateUnweightedScaleSparseTest.java | 164 -
.../UnivariateWeightedScaleDenseTest.java | 161 -
.../UnivariateWeightedScaleSparseTest.java | 163 -
.../applications/dml/ApplyTransformDMLTest.java | 39 -
.../applications/dml/ArimaDMLTest.java | 38 -
.../applications/dml/CsplineCGDMLTest.java | 38 -
.../applications/dml/CsplineDSDMLTest.java | 38 -
.../applications/dml/GLMDMLTest.java | 41 -
.../applications/dml/GNMFDMLTest.java | 37 -
.../applications/dml/HITSDMLTest.java | 31 -
.../applications/dml/ID3DMLTest.java | 38 -
.../applications/dml/L2SVMDMLTest.java | 38 -
.../applications/dml/LinearLogRegDMLTest.java | 38 -
.../dml/LinearRegressionDMLTest.java | 38 -
.../dml/MDABivariateStatsDMLTest.java | 38 -
.../applications/dml/MultiClassSVMDMLTest.java | 38 -
.../applications/dml/NaiveBayesDMLTest.java | 38 -
.../applications/dml/PageRankDMLTest.java | 38 -
.../applications/dml/WelchTDMLTest.java | 38 -
.../parfor/ParForBivariateStatsTest.java | 199 -
.../parfor/ParForCVMulticlassSVMTest.java | 179 -
.../parfor/ParForCorrelationTest.java | 209 -
.../parfor/ParForCorrelationTestLarge.java | 136 -
.../parfor/ParForNaiveBayesTest.java | 217 -
.../parfor/ParForUnivariateStatsTest.java | 182 -
.../pydml/ApplyTransformPyDMLTest.java | 39 -
.../applications/pydml/ArimaPyDMLTest.java | 38 -
.../applications/pydml/CsplineCGPyDMLTest.java | 38 -
.../applications/pydml/CsplineDSPyDMLTest.java | 38 -
.../applications/pydml/GLMPyDMLTest.java | 41 -
.../applications/pydml/GNMFPyDMLTest.java | 37 -
.../applications/pydml/HITSPyDMLTest.java | 31 -
.../applications/pydml/ID3PyDMLTest.java | 38 -
.../applications/pydml/L2SVMPyDMLTest.java | 38 -
.../pydml/LinearLogRegPyDMLTest.java | 38 -
.../pydml/LinearRegressionPyDMLTest.java | 38 -
.../pydml/MDABivariateStatsPyDMLTest.java | 37 -
.../pydml/MultiClassSVMPyDMLTest.java | 38 -
.../applications/pydml/NaiveBayesPyDMLTest.java | 38 -
.../applications/pydml/PageRankPyDMLTest.java | 38 -
.../applications/pydml/WelchTPyDMLTest.java | 38 -
.../functions/aggregate/AggregateInfTest.java | 150 -
.../functions/aggregate/ColSumTest.java | 107 -
.../functions/aggregate/ColSumsSqTest.java | 268 -
.../functions/aggregate/FullAggregateTest.java | 580 --
.../aggregate/FullColAggregateTest.java | 638 --
.../aggregate/FullGroupedAggregateTest.java | 620 --
.../aggregate/FullRowAggregateTest.java | 1186 ----
.../functions/aggregate/LengthTest.java | 98 -
.../functions/aggregate/MaxTest.java | 108 -
.../functions/aggregate/MinTest.java | 108 -
.../functions/aggregate/NColTest.java | 98 -
.../functions/aggregate/NRowTest.java | 98 -
.../functions/aggregate/ProdTest.java | 108 -
.../functions/aggregate/RowSumTest.java | 103 -
.../functions/aggregate/RowSumsSqTest.java | 268 -
.../functions/aggregate/SumSqTest.java | 268 -
.../functions/aggregate/SumTest.java | 107 -
.../functions/aggregate/TraceTest.java | 98 -
.../functions/append/AppendChainTest.java | 195 -
.../functions/append/AppendMatrixTest.java | 279 -
.../functions/append/AppendVectorTest.java | 147 -
.../functions/append/RBindCBindMatrixTest.java | 190 -
.../functions/append/StringAppendTest.java | 143 -
.../binary/matrix/BinUaggChainTest.java | 172 -
.../binary/matrix/CentralMomentTest.java | 217 -
.../functions/binary/matrix/CovarianceTest.java | 146 -
.../matrix/DiagMatrixMultiplicationTest.java | 324 -
.../ElementwiseAdditionMultiplicationTest.java | 137 -
.../binary/matrix/ElementwiseAdditionTest.java | 265 -
.../binary/matrix/ElementwiseDivisionTest.java | 296 -
.../binary/matrix/ElementwiseModulusTest.java | 295 -
.../matrix/ElementwiseMultiplicationTest.java | 266 -
.../matrix/ElementwiseSubtractionTest.java | 295 -
.../binary/matrix/MapMultChainTest.java | 288 -
.../binary/matrix/MapMultLimitTest.java | 100 -
.../binary/matrix/MatrixMultiplicationTest.java | 153 -
.../binary/matrix/MatrixVectorTest.java | 160 -
.../binary/matrix/OuterProductTest.java | 160 -
.../functions/binary/matrix/QuantileTest.java | 294 -
.../binary/matrix/ScalarAdditionTest.java | 247 -
.../binary/matrix/ScalarDivisionTest.java | 327 -
.../binary/matrix/ScalarModulusTest.java | 331 -
.../binary/matrix/ScalarMultiplicationTest.java | 249 -
.../binary/matrix/ScalarSubtractionTest.java | 276 -
.../TransposeMatrixMultiplicationTest.java | 334 -
.../binary/matrix/UaggOuterChainTest.java | 1344 ----
.../UltraSparseMRMatrixMultiplicationTest.java | 211 -
.../ZipMMSparkMatrixMultiplicationTest.java | 174 -
.../FullMatrixMatrixCellwiseOperationTest.java | 821 ---
...ullMatrixVectorColCellwiseOperationTest.java | 818 ---
...ullMatrixVectorRowCellwiseOperationTest.java | 818 ---
.../FullMinus1MultTest.java | 171 -
...ectorVectorCellwiseCompareOperationTest.java | 241 -
.../FullVectorVectorCellwiseOperationTest.java | 798 ---
...FullDistributedMatrixMultiplicationTest.java | 289 -
.../FullIntegerDivisionTest.java | 442 --
.../FullMatrixMultiplicationTest.java | 492 --
...llMatrixMultiplicationTransposeSelfTest.java | 336 -
...FullMatrixMultiplicationUltraSparseTest.java | 200 -
.../FullMinMaxComparisonTest.java | 393 --
.../matrix_full_other/FullPPredMatrixTest.java | 588 --
.../FullPPredScalarLeftTest.java | 435 --
.../FullPPredScalarRightTest.java | 434 --
.../binary/matrix_full_other/FullPowerTest.java | 325 -
.../functions/binary/scalar/AdditionTest.java | 252 -
.../functions/binary/scalar/AndTest.java | 58 -
.../functions/binary/scalar/DivisionTest.java | 345 -
.../functions/binary/scalar/EqualTest.java | 57 -
.../binary/scalar/FullStringComparisonTest.java | 175 -
.../binary/scalar/GreaterThanOrEqualTest.java | 57 -
.../binary/scalar/GreaterThanTest.java | 57 -
.../binary/scalar/LessThanOrEqualTest.java | 57 -
.../functions/binary/scalar/LessThanTest.java | 59 -
.../functions/binary/scalar/LogarithmTest.java | 221 -
.../functions/binary/scalar/ModulusTest.java | 308 -
.../binary/scalar/MultiplicationTest.java | 221 -
.../functions/binary/scalar/OrTest.java | 59 -
.../functions/binary/scalar/PowerTest.java | 221 -
.../binary/scalar/SubtractionTest.java | 482 --
.../functions/blocks/VariableTest.java | 169 -
.../integration/functions/blocks/WhileTest.java | 100 -
.../caching/CachingPWriteExportTest.java | 120 -
.../functions/data/FullReblockTest.java | 527 --
.../data/FullStringInitializeTest.java | 379 --
.../functions/data/RandRuntimePlatformTest.java | 257 -
.../integration/functions/data/RandTest1.java | 94 -
.../integration/functions/data/RandTest2.java | 76 -
.../integration/functions/data/RandTest3.java | 76 -
.../integration/functions/data/RandTest4.java | 84 -
.../integration/functions/data/RandTest5.java | 242 -
.../functions/data/RandVarMinMaxTest.java | 157 -
.../functions/data/RandVarSeedTest.java | 129 -
.../integration/functions/data/ReadMMTest.java | 369 -
.../integration/functions/data/ReblockTest.java | 70 -
.../integration/functions/data/SampleTest.java | 162 -
.../functions/data/SequenceTest.java | 177 -
.../functions/data/VariableTest.java | 73 -
.../integration/functions/data/WriteMMTest.java | 148 -
.../integration/functions/data/WriteTest.java | 134 -
.../functions/dmlscript/DMLScriptTest1.java | 155 -
.../functions/dmlscript/DMLScriptTest2.java | 192 -
.../functions/external/DynProjectTest.java | 135 -
.../functions/external/DynReadWriteTest.java | 103 -
.../external/FunctionExpressionsTest.java | 115 -
.../functions/external/OrderTest.java | 110 -
.../functions/gdfo/GDFOLinregCG.java | 164 -
.../functions/gdfo/GDFOLinregDS.java | 160 -
.../functions/gdfo/GDFOLinregDSsimpl.java | 161 -
.../functions/gdfo/GDFOMMChainLoop.java | 122 -
.../gdfo/HashInterestingPropertiesTest.java | 94 -
.../indexing/Jdk7IssueRightIndexingTest.java | 154 -
.../indexing/LeftIndexingScalarTest.java | 115 -
.../indexing/LeftIndexingSparseDenseTest.java | 209 -
.../indexing/LeftIndexingSparseSparseTest.java | 209 -
.../functions/indexing/LeftIndexingTest.java | 173 -
.../indexing/RightIndexingMatrixTest.java | 171 -
.../indexing/RightIndexingVectorTest.java | 129 -
.../functions/io/FullDynWriteTest.java | 261 -
.../test/integration/functions/io/IOTest1.java | 82 -
.../test/integration/functions/io/IOTest2.java | 78 -
.../test/integration/functions/io/IOTest3.java | 78 -
.../test/integration/functions/io/IOTest4.java | 78 -
.../test/integration/functions/io/IOTest5.java | 78 -
.../integration/functions/io/ScalarIOTest.java | 253 -
.../functions/io/SeqParReadTest.java | 285 -
.../functions/io/binary/SerializeTest.java | 128 -
.../functions/io/csv/CSVParametersTest.java | 223 -
.../functions/io/csv/FormatChangeTest.java | 254 -
.../functions/io/csv/ReadCSVTest.java | 177 -
.../functions/io/csv/WriteCSVTest.java | 133 -
.../io/matrixmarket/FormatChangeTest.java | 169 -
.../functions/io/matrixmarket/ReadMMTest.java | 130 -
.../jmlc/SystemTMulticlassSVMScoreTest.java | 215 -
.../functions/misc/ConditionalValidateTest.java | 150 -
.../functions/misc/DataTypeCastingTest.java | 140 -
.../functions/misc/DataTypeChangeTest.java | 220 -
.../functions/misc/FunctionInliningTest.java | 138 -
.../misc/IPALiteralReplacementTest.java | 109 -
.../functions/misc/IPAScalarRecursionTest.java | 64 -
.../functions/misc/IPAUnknownRecursionTest.java | 94 -
.../misc/InvalidFunctionSignatureTest.java | 67 -
.../functions/misc/LongOverflowTest.java | 119 -
.../functions/misc/NrowNcolStringTest.java | 89 -
.../misc/NrowNcolUnknownCSVReadTest.java | 116 -
.../functions/misc/OuterTableExpandTest.java | 273 -
.../functions/misc/PrintExpressionTest.java | 96 -
.../functions/misc/PrintMatrixTest.java | 60 -
.../functions/misc/ReadAfterWriteTest.java | 135 -
.../RewriteSimplifyRowColSumMVMultTest.java | 124 -
.../misc/RewriteSlicedMatrixMultTest.java | 123 -
.../functions/misc/ScalarAssignmentTest.java | 191 -
.../functions/misc/ScalarFunctionTest.java | 93 -
.../functions/misc/SetWorkingDirTest.java | 132 -
.../misc/ValueTypeAutoCastingTest.java | 185 -
.../functions/misc/ValueTypeCastingTest.java | 200 -
.../functions/mlcontext/GNMFTest.java | 225 -
.../functions/parfor/ForLoopPredicateTest.java | 184 -
.../parfor/ParForAdversarialLiteralsTest.java | 146 -
.../ParForColwiseDataPartitioningTest.java | 368 -
.../ParForDataPartitionLeftIndexingTest.java | 167 -
.../parfor/ParForDependencyAnalysisTest.java | 374 --
.../parfor/ParForFunctionSerializationTest.java | 103 -
.../ParForMultipleDataPartitioningTest.java | 108 -
.../parfor/ParForNaNResultMergeTest.java | 133 -
.../ParForParallelRemoteResultMergeTest.java | 135 -
.../parfor/ParForRepeatedOptimizationTest.java | 175 -
.../ParForReplaceThreadIDRecompileTest.java | 99 -
.../ParForRowwiseDataPartitioningTest.java | 369 -
.../parfor/ParForRulebasedOptimizerTest.java | 316 -
.../ParForSerialRemoteResultMergeTest.java | 181 -
.../piggybacking/PiggybackingTest1.java | 153 -
.../piggybacking/PiggybackingTest2.java | 79 -
.../quaternary/RewritesWeightedSigmoidTest.java | 161 -
.../quaternary/WeightedCrossEntropyTest.java | 186 -
.../quaternary/WeightedDivMatrixMultTest.java | 496 --
.../quaternary/WeightedSigmoidTest.java | 499 --
.../quaternary/WeightedSquaredLossTest.java | 510 --
.../quaternary/WeightedUnaryMatrixMultTest.java | 284 -
.../functions/recompile/BranchRemovalTest.java | 169 -
.../recompile/CSVReadUnknownSizeTest.java | 160 -
.../recompile/FunctionRecompileTest.java | 139 -
.../IPAAssignConstantPropagationTest.java | 126 -
.../recompile/IPAComplexAppendTest.java | 133 -
.../recompile/IPAConstantPropagationTest.java | 150 -
...IPAPropagationSizeMultipleFunctionsTest.java | 170 -
.../LiteralReplaceCastScalarReadTest.java | 102 -
.../recompile/MultipleReadsIPATest.java | 131 -
.../recompile/PredicateRecompileTest.java | 362 -
.../recompile/RandJobRecompileTest.java | 124 -
.../functions/recompile/RandRecompileTest.java | 170 -
.../recompile/RandSizeExpressionEvalTest.java | 125 -
.../recompile/ReblockRecompileTest.java | 136 -
.../recompile/RemoveEmptyPotpourriTest.java | 135 -
.../recompile/RemoveEmptyRecompileTest.java | 324 -
.../RewriteComplexMapMultChainTest.java | 177 -
.../SparsityFunctionRecompileTest.java | 228 -
.../recompile/SparsityRecompileTest.java | 188 -
.../functions/reorg/DiagV2MTest.java | 113 -
.../functions/reorg/FullOrderTest.java | 993 ---
.../functions/reorg/FullTransposeTest.java | 244 -
.../functions/reorg/MatrixReshapeTest.java | 393 --
.../terms/ScalarMatrixUnaryBinaryTermTest.java | 62 -
.../terms/ScalarToMatrixInLoopTest.java | 47 -
.../ternary/CTableMatrixIgnoreZerosTest.java | 181 -
.../functions/ternary/CTableSequenceTest.java | 299 -
.../ternary/CentralMomentWeightsTest.java | 221 -
.../ternary/CovarianceWeightsTest.java | 151 -
.../functions/ternary/QuantileWeightsTest.java | 296 -
.../functions/ternary/TableOutputTest.java | 216 -
.../functions/transform/RunTest.java | 266 -
.../functions/transform/ScalingTest.java | 243 -
.../functions/transform/TransformTest.java | 708 --
.../functions/unary/matrix/ACosTest.java | 148 -
.../functions/unary/matrix/ASinTest.java | 146 -
.../functions/unary/matrix/ATanTest.java | 148 -
.../functions/unary/matrix/AbsTest.java | 146 -
.../unary/matrix/CastAsScalarTest.java | 55 -
.../functions/unary/matrix/CosTest.java | 146 -
.../functions/unary/matrix/DiagTest.java | 106 -
.../unary/matrix/EigenFactorizeTest.java | 133 -
.../functions/unary/matrix/FullCummaxTest.java | 276 -
.../functions/unary/matrix/FullCumminTest.java | 276 -
.../functions/unary/matrix/FullCumprodTest.java | 277 -
.../functions/unary/matrix/FullCumsumTest.java | 275 -
.../unary/matrix/FullSelectPosTest.java | 187 -
.../functions/unary/matrix/IQMTest.java | 296 -
.../functions/unary/matrix/LUFactorizeTest.java | 129 -
.../unary/matrix/MLUnaryBuiltinTest.java | 273 -
.../unary/matrix/MatrixInverseTest.java | 100 -
.../functions/unary/matrix/MinusTest.java | 138 -
.../functions/unary/matrix/NegationTest.java | 145 -
.../functions/unary/matrix/PrintTest.java | 66 -
.../functions/unary/matrix/QRSolverTest.java | 119 -
.../functions/unary/matrix/RemoveEmptyTest.java | 616 --
.../functions/unary/matrix/ReplaceTest.java | 360 -
.../functions/unary/matrix/RoundTest.java | 495 --
.../functions/unary/matrix/SinTest.java | 146 -
.../functions/unary/matrix/SqrtTest.java | 112 -
.../functions/unary/matrix/TanTest.java | 146 -
.../functions/unary/matrix/TransposeTest.java | 74 -
.../functions/unary/scalar/ACosTest.java | 155 -
.../functions/unary/scalar/ASinTest.java | 127 -
.../functions/unary/scalar/ATanTest.java | 128 -
.../functions/unary/scalar/AbsTest.java | 152 -
.../functions/unary/scalar/BooleanTest.java | 59 -
.../functions/unary/scalar/CosTest.java | 154 -
.../functions/unary/scalar/ExponentTest.java | 210 -
.../unary/scalar/FullDistributionTest.java | 133 -
.../functions/unary/scalar/NegationTest.java | 78 -
.../functions/unary/scalar/NotTest.java | 66 -
.../functions/unary/scalar/PrintTest.java | 117 -
.../functions/unary/scalar/RoundTest.java | 72 -
.../functions/unary/scalar/SinTest.java | 128 -
.../functions/unary/scalar/SqrtTest.java | 155 -
.../functions/unary/scalar/StopTest.java | 75 -
.../functions/unary/scalar/StopTest2.java | 68 -
.../functions/unary/scalar/StopTestCtrlStr.java | 133 -
.../functions/unary/scalar/TanTest.java | 128 -
.../functions/vect/AutoVectorizationTest.java | 255 -
.../scalability/LinearRegressionTest.java | 53 -
.../integration/scalability/PageRankTest.java | 53 -
.../com/ibm/bi/dml/test/utils/TestUtils.java | 2262 -------
.../AutomatedScalabilityTestBase.java | 91 +
.../test/integration/AutomatedTestBase.java | 1627 +++++
.../BinaryMatrixCharacteristics.java | 84 +
.../test/integration/TestConfiguration.java | 174 +
.../integration/TestMatrixCharacteristics.java | 100 +
.../applications/ApplyTransformTest.java | 176 +
.../integration/applications/ArimaTest.java | 119 +
.../integration/applications/CsplineCGTest.java | 103 +
.../integration/applications/CsplineDSTest.java | 110 +
.../test/integration/applications/GLMTest.java | 697 ++
.../test/integration/applications/GNMFTest.java | 133 +
.../test/integration/applications/HITSTest.java | 88 +
.../test/integration/applications/ID3Test.java | 122 +
.../integration/applications/L2SVMTest.java | 116 +
.../applications/LinearLogRegTest.java | 114 +
.../applications/LinearRegressionTest.java | 109 +
.../applications/MDABivariateStatsTest.java | 133 +
.../applications/MultiClassSVMTest.java | 138 +
.../applications/NaiveBayesTest.java | 122 +
.../integration/applications/PageRankTest.java | 112 +
.../integration/applications/WelchTTest.java | 104 +
.../BivariateCategoricalCategoricallTest.java | 256 +
.../BivariateOrdinalOrdinalTest.java | 161 +
.../BivariateScaleCategoricalTest.java | 181 +
.../BivariateScaleScaleTest.java | 177 +
.../descriptivestats/OrderStatisticsTest.java | 158 +
.../UnivariateCategoricalTest.java | 145 +
.../descriptivestats/UnivariateStatsBase.java | 294 +
.../UnivariateUnweightedScaleDenseTest.java | 164 +
.../UnivariateUnweightedScaleSparseTest.java | 164 +
.../UnivariateWeightedScaleDenseTest.java | 161 +
.../UnivariateWeightedScaleSparseTest.java | 163 +
.../applications/dml/ApplyTransformDMLTest.java | 39 +
.../applications/dml/ArimaDMLTest.java | 38 +
.../applications/dml/CsplineCGDMLTest.java | 38 +
.../applications/dml/CsplineDSDMLTest.java | 38 +
.../applications/dml/GLMDMLTest.java | 41 +
.../applications/dml/GNMFDMLTest.java | 37 +
.../applications/dml/HITSDMLTest.java | 31 +
.../applications/dml/ID3DMLTest.java | 38 +
.../applications/dml/L2SVMDMLTest.java | 38 +
.../applications/dml/LinearLogRegDMLTest.java | 38 +
.../dml/LinearRegressionDMLTest.java | 38 +
.../dml/MDABivariateStatsDMLTest.java | 38 +
.../applications/dml/MultiClassSVMDMLTest.java | 38 +
.../applications/dml/NaiveBayesDMLTest.java | 38 +
.../applications/dml/PageRankDMLTest.java | 38 +
.../applications/dml/WelchTDMLTest.java | 38 +
.../parfor/ParForBivariateStatsTest.java | 199 +
.../parfor/ParForCVMulticlassSVMTest.java | 179 +
.../parfor/ParForCorrelationTest.java | 209 +
.../parfor/ParForCorrelationTestLarge.java | 136 +
.../parfor/ParForNaiveBayesTest.java | 217 +
.../parfor/ParForUnivariateStatsTest.java | 182 +
.../pydml/ApplyTransformPyDMLTest.java | 39 +
.../applications/pydml/ArimaPyDMLTest.java | 38 +
.../applications/pydml/CsplineCGPyDMLTest.java | 38 +
.../applications/pydml/CsplineDSPyDMLTest.java | 38 +
.../applications/pydml/GLMPyDMLTest.java | 41 +
.../applications/pydml/GNMFPyDMLTest.java | 37 +
.../applications/pydml/HITSPyDMLTest.java | 31 +
.../applications/pydml/ID3PyDMLTest.java | 38 +
.../applications/pydml/L2SVMPyDMLTest.java | 38 +
.../pydml/LinearLogRegPyDMLTest.java | 38 +
.../pydml/LinearRegressionPyDMLTest.java | 38 +
.../pydml/MDABivariateStatsPyDMLTest.java | 37 +
.../pydml/MultiClassSVMPyDMLTest.java | 38 +
.../applications/pydml/NaiveBayesPyDMLTest.java | 38 +
.../applications/pydml/PageRankPyDMLTest.java | 38 +
.../applications/pydml/WelchTPyDMLTest.java | 38 +
.../functions/aggregate/AggregateInfTest.java | 150 +
.../functions/aggregate/ColSumTest.java | 107 +
.../functions/aggregate/ColSumsSqTest.java | 268 +
.../functions/aggregate/FullAggregateTest.java | 580 ++
.../aggregate/FullColAggregateTest.java | 638 ++
.../aggregate/FullGroupedAggregateTest.java | 620 ++
.../aggregate/FullRowAggregateTest.java | 1186 ++++
.../functions/aggregate/LengthTest.java | 98 +
.../functions/aggregate/MaxTest.java | 108 +
.../functions/aggregate/MinTest.java | 108 +
.../functions/aggregate/NColTest.java | 98 +
.../functions/aggregate/NRowTest.java | 98 +
.../functions/aggregate/ProdTest.java | 108 +
.../functions/aggregate/RowSumTest.java | 103 +
.../functions/aggregate/RowSumsSqTest.java | 268 +
.../functions/aggregate/SumSqTest.java | 268 +
.../functions/aggregate/SumTest.java | 107 +
.../functions/aggregate/TraceTest.java | 98 +
.../functions/append/AppendChainTest.java | 195 +
.../functions/append/AppendMatrixTest.java | 279 +
.../functions/append/AppendVectorTest.java | 147 +
.../functions/append/RBindCBindMatrixTest.java | 190 +
.../functions/append/StringAppendTest.java | 143 +
.../binary/matrix/BinUaggChainTest.java | 172 +
.../binary/matrix/CentralMomentTest.java | 217 +
.../functions/binary/matrix/CovarianceTest.java | 146 +
.../matrix/DiagMatrixMultiplicationTest.java | 324 +
.../ElementwiseAdditionMultiplicationTest.java | 137 +
.../binary/matrix/ElementwiseAdditionTest.java | 265 +
.../binary/matrix/ElementwiseDivisionTest.java | 296 +
.../binary/matrix/ElementwiseModulusTest.java | 295 +
.../matrix/ElementwiseMultiplicationTest.java | 266 +
.../matrix/ElementwiseSubtractionTest.java | 295 +
.../binary/matrix/MapMultChainTest.java | 288 +
.../binary/matrix/MapMultLimitTest.java | 100 +
.../binary/matrix/MatrixMultiplicationTest.java | 153 +
.../binary/matrix/MatrixVectorTest.java | 160 +
.../binary/matrix/OuterProductTest.java | 160 +
.../functions/binary/matrix/QuantileTest.java | 294 +
.../binary/matrix/ScalarAdditionTest.java | 247 +
.../binary/matrix/ScalarDivisionTest.java | 327 +
.../binary/matrix/ScalarModulusTest.java | 331 +
.../binary/matrix/ScalarMultiplicationTest.java | 249 +
.../binary/matrix/ScalarSubtractionTest.java | 276 +
.../TransposeMatrixMultiplicationTest.java | 334 +
.../binary/matrix/UaggOuterChainTest.java | 1344 ++++
.../UltraSparseMRMatrixMultiplicationTest.java | 211 +
.../ZipMMSparkMatrixMultiplicationTest.java | 174 +
.../FullMatrixMatrixCellwiseOperationTest.java | 821 +++
...ullMatrixVectorColCellwiseOperationTest.java | 818 +++
...ullMatrixVectorRowCellwiseOperationTest.java | 818 +++
.../FullMinus1MultTest.java | 171 +
...ectorVectorCellwiseCompareOperationTest.java | 241 +
.../FullVectorVectorCellwiseOperationTest.java | 798 +++
...FullDistributedMatrixMultiplicationTest.java | 289 +
.../FullIntegerDivisionTest.java | 442 ++
.../FullMatrixMultiplicationTest.java | 492 ++
...llMatrixMultiplicationTransposeSelfTest.java | 336 +
...FullMatrixMultiplicationUltraSparseTest.java | 200 +
.../FullMinMaxComparisonTest.java | 393 ++
.../matrix_full_other/FullPPredMatrixTest.java | 588 ++
.../FullPPredScalarLeftTest.java | 435 ++
.../FullPPredScalarRightTest.java | 434 ++
.../binary/matrix_full_other/FullPowerTest.java | 325 +
.../functions/binary/scalar/AdditionTest.java | 252 +
.../functions/binary/scalar/AndTest.java | 58 +
.../functions/binary/scalar/DivisionTest.java | 345 +
.../functions/binary/scalar/EqualTest.java | 57 +
.../binary/scalar/FullStringComparisonTest.java | 175 +
.../binary/scalar/GreaterThanOrEqualTest.java | 57 +
.../binary/scalar/GreaterThanTest.java | 57 +
.../binary/scalar/LessThanOrEqualTest.java | 57 +
.../functions/binary/scalar/LessThanTest.java | 59 +
.../functions/binary/scalar/LogarithmTest.java | 221 +
.../functions/binary/scalar/ModulusTest.java | 308 +
.../binary/scalar/MultiplicationTest.java | 221 +
.../functions/binary/scalar/OrTest.java | 59 +
.../functions/binary/scalar/PowerTest.java | 221 +
.../binary/scalar/SubtractionTest.java | 482 ++
.../functions/blocks/VariableTest.java | 169 +
.../integration/functions/blocks/WhileTest.java | 100 +
.../caching/CachingPWriteExportTest.java | 120 +
.../functions/data/FullReblockTest.java | 527 ++
.../data/FullStringInitializeTest.java | 379 ++
.../functions/data/RandRuntimePlatformTest.java | 257 +
.../integration/functions/data/RandTest1.java | 94 +
.../integration/functions/data/RandTest2.java | 76 +
.../integration/functions/data/RandTest3.java | 76 +
.../integration/functions/data/RandTest4.java | 84 +
.../integration/functions/data/RandTest5.java | 242 +
.../functions/data/RandVarMinMaxTest.java | 157 +
.../functions/data/RandVarSeedTest.java | 129 +
.../integration/functions/data/ReadMMTest.java | 369 +
.../integration/functions/data/ReblockTest.java | 70 +
.../integration/functions/data/SampleTest.java | 162 +
.../functions/data/SequenceTest.java | 177 +
.../functions/data/VariableTest.java | 73 +
.../integration/functions/data/WriteMMTest.java | 148 +
.../integration/functions/data/WriteTest.java | 134 +
.../functions/dmlscript/DMLScriptTest1.java | 155 +
.../functions/dmlscript/DMLScriptTest2.java | 192 +
.../functions/external/DynProjectTest.java | 135 +
.../functions/external/DynReadWriteTest.java | 103 +
.../external/FunctionExpressionsTest.java | 115 +
.../functions/external/OrderTest.java | 110 +
.../functions/gdfo/GDFOLinregCG.java | 164 +
.../functions/gdfo/GDFOLinregDS.java | 160 +
.../functions/gdfo/GDFOLinregDSsimpl.java | 161 +
.../functions/gdfo/GDFOMMChainLoop.java | 122 +
.../gdfo/HashInterestingPropertiesTest.java | 94 +
.../indexing/Jdk7IssueRightIndexingTest.java | 154 +
.../indexing/LeftIndexingScalarTest.java | 115 +
.../indexing/LeftIndexingSparseDenseTest.java | 209 +
.../indexing/LeftIndexingSparseSparseTest.java | 209 +
.../functions/indexing/LeftIndexingTest.java | 173 +
.../indexing/RightIndexingMatrixTest.java | 171 +
.../indexing/RightIndexingVectorTest.java | 129 +
.../functions/io/FullDynWriteTest.java | 261 +
.../test/integration/functions/io/IOTest1.java | 82 +
.../test/integration/functions/io/IOTest2.java | 78 +
.../test/integration/functions/io/IOTest3.java | 78 +
.../test/integration/functions/io/IOTest4.java | 78 +
.../test/integration/functions/io/IOTest5.java | 78 +
.../integration/functions/io/ScalarIOTest.java | 253 +
.../functions/io/SeqParReadTest.java | 285 +
.../functions/io/binary/SerializeTest.java | 128 +
.../functions/io/csv/CSVParametersTest.java | 223 +
.../functions/io/csv/FormatChangeTest.java | 254 +
.../functions/io/csv/ReadCSVTest.java | 177 +
.../functions/io/csv/WriteCSVTest.java | 133 +
.../io/matrixmarket/FormatChangeTest.java | 169 +
.../functions/io/matrixmarket/ReadMMTest.java | 130 +
.../jmlc/SystemTMulticlassSVMScoreTest.java | 215 +
.../functions/misc/ConditionalValidateTest.java | 150 +
.../functions/misc/DataTypeCastingTest.java | 140 +
.../functions/misc/DataTypeChangeTest.java | 220 +
.../functions/misc/FunctionInliningTest.java | 138 +
.../misc/IPALiteralReplacementTest.java | 109 +
.../functions/misc/IPAScalarRecursionTest.java | 64 +
.../functions/misc/IPAUnknownRecursionTest.java | 94 +
.../misc/InvalidFunctionSignatureTest.java | 67 +
.../functions/misc/LongOverflowTest.java | 119 +
.../functions/misc/NrowNcolStringTest.java | 89 +
.../misc/NrowNcolUnknownCSVReadTest.java | 116 +
.../functions/misc/OuterTableExpandTest.java | 273 +
.../functions/misc/PrintExpressionTest.java | 96 +
.../functions/misc/PrintMatrixTest.java | 60 +
.../functions/misc/ReadAfterWriteTest.java | 135 +
.../RewriteSimplifyRowColSumMVMultTest.java | 124 +
.../misc/RewriteSlicedMatrixMultTest.java | 123 +
.../functions/misc/ScalarAssignmentTest.java | 191 +
.../functions/misc/ScalarFunctionTest.java | 93 +
.../functions/misc/SetWorkingDirTest.java | 132 +
.../misc/ValueTypeAutoCastingTest.java | 185 +
.../functions/misc/ValueTypeCastingTest.java | 200 +
.../functions/mlcontext/GNMFTest.java | 225 +
.../functions/parfor/ForLoopPredicateTest.java | 184 +
.../parfor/ParForAdversarialLiteralsTest.java | 146 +
.../ParForColwiseDataPartitioningTest.java | 368 +
.../ParForDataPartitionLeftIndexingTest.java | 167 +
.../parfor/ParForDependencyAnalysisTest.java | 374 ++
.../parfor/ParForFunctionSerializationTest.java | 103 +
.../ParForMultipleDataPartitioningTest.java | 108 +
.../parfor/ParForNaNResultMergeTest.java | 133 +
.../ParForParallelRemoteResultMergeTest.java | 135 +
.../parfor/ParForRepeatedOptimizationTest.java | 175 +
.../ParForReplaceThreadIDRecompileTest.java | 99 +
.../ParForRowwiseDataPartitioningTest.java | 369 +
.../parfor/ParForRulebasedOptimizerTest.java | 316 +
.../ParForSerialRemoteResultMergeTest.java | 181 +
.../piggybacking/PiggybackingTest1.java | 153 +
.../piggybacking/PiggybackingTest2.java | 79 +
.../quaternary/RewritesWeightedSigmoidTest.java | 161 +
.../quaternary/WeightedCrossEntropyTest.java | 186 +
.../quaternary/WeightedDivMatrixMultTest.java | 496 ++
.../quaternary/WeightedSigmoidTest.java | 499 ++
.../quaternary/WeightedSquaredLossTest.java | 510 ++
.../quaternary/WeightedUnaryMatrixMultTest.java | 284 +
.../functions/recompile/BranchRemovalTest.java | 169 +
.../recompile/CSVReadUnknownSizeTest.java | 160 +
.../recompile/FunctionRecompileTest.java | 139 +
.../IPAAssignConstantPropagationTest.java | 126 +
.../recompile/IPAComplexAppendTest.java | 133 +
.../recompile/IPAConstantPropagationTest.java | 150 +
...IPAPropagationSizeMultipleFunctionsTest.java | 170 +
.../LiteralReplaceCastScalarReadTest.java | 102 +
.../recompile/MultipleReadsIPATest.java | 131 +
.../recompile/PredicateRecompileTest.java | 362 +
.../recompile/RandJobRecompileTest.java | 124 +
.../functions/recompile/RandRecompileTest.java | 170 +
.../recompile/RandSizeExpressionEvalTest.java | 125 +
.../recompile/ReblockRecompileTest.java | 136 +
.../recompile/RemoveEmptyPotpourriTest.java | 135 +
.../recompile/RemoveEmptyRecompileTest.java | 324 +
.../RewriteComplexMapMultChainTest.java | 177 +
.../SparsityFunctionRecompileTest.java | 228 +
.../recompile/SparsityRecompileTest.java | 188 +
.../functions/reorg/DiagV2MTest.java | 113 +
.../functions/reorg/FullOrderTest.java | 993 +++
.../functions/reorg/FullTransposeTest.java | 244 +
.../functions/reorg/MatrixReshapeTest.java | 393 ++
.../terms/ScalarMatrixUnaryBinaryTermTest.java | 62 +
.../terms/ScalarToMatrixInLoopTest.java | 47 +
.../ternary/CTableMatrixIgnoreZerosTest.java | 181 +
.../functions/ternary/CTableSequenceTest.java | 299 +
.../ternary/CentralMomentWeightsTest.java | 221 +
.../ternary/CovarianceWeightsTest.java | 151 +
.../functions/ternary/QuantileWeightsTest.java | 296 +
.../functions/ternary/TableOutputTest.java | 216 +
.../functions/transform/RunTest.java | 266 +
.../functions/transform/ScalingTest.java | 243 +
.../functions/transform/TransformTest.java | 708 ++
.../functions/unary/matrix/ACosTest.java | 148 +
.../functions/unary/matrix/ASinTest.java | 146 +
.../functions/unary/matrix/ATanTest.java | 148 +
.../functions/unary/matrix/AbsTest.java | 146 +
.../unary/matrix/CastAsScalarTest.java | 55 +
.../functions/unary/matrix/CosTest.java | 146 +
.../functions/unary/matrix/DiagTest.java | 106 +
.../unary/matrix/EigenFactorizeTest.java | 133 +
.../functions/unary/matrix/FullCummaxTest.java | 276 +
.../functions/unary/matrix/FullCumminTest.java | 276 +
.../functions/unary/matrix/FullCumprodTest.java | 277 +
.../functions/unary/matrix/FullCumsumTest.java | 275 +
.../unary/matrix/FullSelectPosTest.java | 187 +
.../functions/unary/matrix/IQMTest.java | 296 +
.../functions/unary/matrix/LUFactorizeTest.java | 129 +
.../unary/matrix/MLUnaryBuiltinTest.java | 273 +
.../unary/matrix/MatrixInverseTest.java | 100 +
.../functions/unary/matrix/MinusTest.java | 138 +
.../functions/unary/matrix/NegationTest.java | 145 +
.../functions/unary/matrix/PrintTest.java | 66 +
.../functions/unary/matrix/QRSolverTest.java | 119 +
.../functions/unary/matrix/RemoveEmptyTest.java | 616 ++
.../functions/unary/matrix/ReplaceTest.java | 360 +
.../functions/unary/matrix/RoundTest.java | 495 ++
.../functions/unary/matrix/SinTest.java | 146 +
.../functions/unary/matrix/SqrtTest.java | 112 +
.../functions/unary/matrix/TanTest.java | 146 +
.../functions/unary/matrix/TransposeTest.java | 74 +
.../functions/unary/scalar/ACosTest.java | 155 +
.../functions/unary/scalar/ASinTest.java | 127 +
.../functions/unary/scalar/ATanTest.java | 128 +
.../functions/unary/scalar/AbsTest.java | 152 +
.../functions/unary/scalar/BooleanTest.java | 59 +
.../functions/unary/scalar/CosTest.java | 154 +
.../functions/unary/scalar/ExponentTest.java | 210 +
.../unary/scalar/FullDistributionTest.java | 133 +
.../functions/unary/scalar/NegationTest.java | 78 +
.../functions/unary/scalar/NotTest.java | 66 +
.../functions/unary/scalar/PrintTest.java | 117 +
.../functions/unary/scalar/RoundTest.java | 72 +
.../functions/unary/scalar/SinTest.java | 128 +
.../functions/unary/scalar/SqrtTest.java | 155 +
.../functions/unary/scalar/StopTest.java | 75 +
.../functions/unary/scalar/StopTest2.java | 68 +
.../functions/unary/scalar/StopTestCtrlStr.java | 133 +
.../functions/unary/scalar/TanTest.java | 128 +
.../functions/vect/AutoVectorizationTest.java | 255 +
.../scalability/LinearRegressionTest.java | 53 +
.../integration/scalability/PageRankTest.java | 53 +
.../org/apache/sysml/test/utils/TestUtils.java | 2262 +++++++
.../integration/applications/ZPackageSuite.java | 71 -
.../descriptivestats/ZPackageSuite.java | 44 -
.../applications/parfor/ZPackageSuite.java | 40 -
.../functions/aggregate/ZPackageSuite.java | 52 -
.../functions/append/ZPackageSuite.java | 38 -
.../functions/binary/matrix/ZPackageSuite.java | 62 -
.../matrix_full_cellwise/ZPackageSuite.java | 39 -
.../binary/matrix_full_other/ZPackageSuite.java | 44 -
.../functions/binary/scalar/ZPackageSuite.java | 48 -
.../functions/blocks/ZPackageSuite.java | 35 -
.../functions/caching/ZPackageSuite.java | 34 -
.../functions/data/ZPackageSuite.java | 51 -
.../functions/dmlscript/ZPackageSuite.java | 35 -
.../functions/external/ZPackageSuite.java | 37 -
.../functions/gdfo/ZPackageSuite.java | 38 -
.../functions/indexing/ZPackageSuite.java | 41 -
.../integration/functions/io/ZPackageSuite.java | 41 -
.../functions/io/binary/ZPackageSuite.java | 34 -
.../functions/io/csv/ZPackageSuite.java | 37 -
.../io/matrixmarket/ZPackageSuite.java | 35 -
.../functions/jmlc/ZPackageSuite.java | 34 -
.../functions/misc/ZPackageSuite.java | 54 -
.../functions/mlcontext/ZPackageSuite.java | 34 -
.../functions/parfor/ZPackageSuite.java | 48 -
.../functions/piggybacking/ZPackageSuite.java | 35 -
.../functions/quaternary/ZPackageSuite.java | 39 -
.../functions/recompile/ZPackageSuite.java | 52 -
.../functions/reorg/ZPackageSuite.java | 38 -
.../functions/terms/ZPackageSuite.java | 35 -
.../functions/ternary/ZPackageSuite.java | 39 -
.../functions/transform/ZPackageSuite.java | 36 -
.../functions/unary/matrix/ZPackageSuite.java | 62 -
.../functions/unary/scalar/ZPackageSuite.java | 51 -
.../functions/vect/ZPackageSuite.java | 34 -
.../integration/scalability/ZPackageSuite.java | 35 -
.../integration/applications/ZPackageSuite.java | 71 +
.../descriptivestats/ZPackageSuite.java | 44 +
.../applications/parfor/ZPackageSuite.java | 40 +
.../functions/aggregate/ZPackageSuite.java | 52 +
.../functions/append/ZPackageSuite.java | 38 +
.../functions/binary/matrix/ZPackageSuite.java | 62 +
.../matrix_full_cellwise/ZPackageSuite.java | 39 +
.../binary/matrix_full_other/ZPackageSuite.java | 44 +
.../functions/binary/scalar/ZPackageSuite.java | 48 +
.../functions/blocks/ZPackageSuite.java | 35 +
.../functions/caching/ZPackageSuite.java | 34 +
.../functions/data/ZPackageSuite.java | 51 +
.../functions/dmlscript/ZPackageSuite.java | 35 +
.../functions/external/ZPackageSuite.java | 37 +
.../functions/gdfo/ZPackageSuite.java | 38 +
.../functions/indexing/ZPackageSuite.java | 41 +
.../integration/functions/io/ZPackageSuite.java | 41 +
.../functions/io/binary/ZPackageSuite.java | 34 +
.../functions/io/csv/ZPackageSuite.java | 37 +
.../io/matrixmarket/ZPackageSuite.java | 35 +
.../functions/jmlc/ZPackageSuite.java | 34 +
.../functions/misc/ZPackageSuite.java | 54 +
.../functions/mlcontext/ZPackageSuite.java | 34 +
.../functions/parfor/ZPackageSuite.java | 48 +
.../functions/piggybacking/ZPackageSuite.java | 35 +
.../functions/quaternary/ZPackageSuite.java | 39 +
.../functions/recompile/ZPackageSuite.java | 52 +
.../functions/reorg/ZPackageSuite.java | 38 +
.../functions/terms/ZPackageSuite.java | 35 +
.../functions/ternary/ZPackageSuite.java | 39 +
.../functions/transform/ZPackageSuite.java | 36 +
.../functions/unary/matrix/ZPackageSuite.java | 62 +
.../functions/unary/scalar/ZPackageSuite.java | 51 +
.../functions/vect/ZPackageSuite.java | 34 +
.../integration/scalability/ZPackageSuite.java | 35 +
2438 files changed, 295773 insertions(+), 295773 deletions(-)
----------------------------------------------------------------------
[40/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/ParameterizedBuiltinOp.java b/src/main/java/com/ibm/bi/dml/hops/ParameterizedBuiltinOp.java
deleted file mode 100644
index 8387736..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/ParameterizedBuiltinOp.java
+++ /dev/null
@@ -1,1139 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.GroupedAggregate;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-/**
- * Defines the HOP for calling an internal function (with custom parameters) from a DML script.
- *
- */
-public class ParameterizedBuiltinOp extends Hop
-{
-
- private static boolean COMPILE_PARALLEL_REMOVEEMPTY = true;
- public static boolean FORCE_DIST_RM_EMPTY = false;
-
- //operator type
- private ParamBuiltinOp _op;
-
- //removeEmpty hints
- private boolean _outputEmptyBlocks = true;
- private boolean _outputPermutationMatrix = false;
-
- private boolean _bRmEmptyBC = false;
-
- /**
- * List of "named" input parameters. They are maintained as a hashmap:
- * parameter names (String) are mapped as indices (Integer) into getInput()
- * arraylist.
- *
- * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
- * that is associated with parameterName.
- */
- private HashMap<String, Integer> _paramIndexMap = new HashMap<String, Integer>();
-
- private ParameterizedBuiltinOp() {
- //default constructor for clone
- }
-
- /**
- * Creates a new HOP for a function call
- */
- public ParameterizedBuiltinOp(String l, DataType dt, ValueType vt,
- ParamBuiltinOp op, HashMap<String, Hop> inputParameters) {
- super(l, dt, vt);
-
- _op = op;
-
- int index = 0;
- for( Entry<String,Hop> e : inputParameters.entrySet() )
- {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- public HashMap<String, Integer> getParamIndexMap(){
- return _paramIndexMap;
- }
-
- @Override
- public String getOpString() {
- return "" + _op;
- }
-
- public ParamBuiltinOp getOp() {
- return _op;
- }
-
- public void setOutputEmptyBlocks(boolean flag)
- {
- _outputEmptyBlocks = flag;
- }
-
- public void setOutputPermutationMatrix(boolean flag)
- {
- _outputPermutationMatrix = flag;
- }
-
- public Hop getTargetHop()
- {
- Hop targetHop = getInput().get(_paramIndexMap.get("target"));
-
- return targetHop;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- // construct lops for all input parameters
- HashMap<String, Lop> inputlops = new HashMap<String, Lop>();
- for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
- inputlops.put(cur.getKey(), getInput().get(cur.getValue())
- .constructLops());
- }
-
- if ( _op == ParamBuiltinOp.CDF || _op == ParamBuiltinOp.INVCDF )
- {
- // simply pass the hashmap of parameters to the lop
- // set the lop for the function call (always CP)
-
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(),
- getValueType());
-
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
- setLops(pbilop);
- }
- else if (_op == ParamBuiltinOp.GROUPEDAGG)
- {
- ExecType et = optFindExecType();
-
- constructLopsGroupedAggregate(inputlops, et);
- }
- else if( _op == ParamBuiltinOp.RMEMPTY )
- {
- ExecType et = optFindExecType();
- et = (et == ExecType.MR && !COMPILE_PARALLEL_REMOVEEMPTY ) ? ExecType.CP_FILE : et;
-
- constructLopsRemoveEmpty(inputlops, et);
- }
- else if( _op == ParamBuiltinOp.REPLACE )
- {
- ExecType et = optFindExecType();
-
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
-
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
- setLops(pbilop);
- }
- else if( _op == ParamBuiltinOp.REXPAND )
- {
- ExecType et = optFindExecType();
-
- constructLopsRExpand(inputlops, et);
- }
- else if ( _op == ParamBuiltinOp.TRANSFORM )
- {
- ExecType et = optFindExecType();
-
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
- // output of transform is always in CSV format
- // to produce a blocked output, this lop must be
- // fed into CSV Reblock lop.
- pbilop.getOutputParameters().setFormat(Format.CSV);
- setLops(pbilop);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- private void constructLopsGroupedAggregate(HashMap<String, Lop> inputlops, ExecType et)
- throws HopsException, LopsException
- {
- //reset reblock requirement (see MR aggregate / construct lops)
- setRequiresReblock( false );
-
- if ( et == ExecType.MR )
- {
- // construct necessary lops: combineBinary/combineTertiary and
- // groupedAgg
-
- boolean isWeighted = (_paramIndexMap.get(Statement.GAGG_WEIGHTS) != null);
- if (isWeighted)
- {
- Lop append = BinaryOp.constructAppendLopChain(
- getInput().get(_paramIndexMap.get(Statement.GAGG_TARGET)),
- getInput().get(_paramIndexMap.get(Statement.GAGG_GROUPS)),
- getInput().get(_paramIndexMap.get(Statement.GAGG_WEIGHTS)),
- DataType.MATRIX, getValueType(), true,
- getInput().get(_paramIndexMap.get(Statement.GAGG_TARGET)));
-
- // add the combine lop to parameter list, with a new name "combinedinput"
- inputlops.put(GroupedAggregate.COMBINEDINPUT, append);
- inputlops.remove(Statement.GAGG_TARGET);
- inputlops.remove(Statement.GAGG_GROUPS);
- inputlops.remove(Statement.GAGG_WEIGHTS);
-
- }
- else
- {
- Lop append = BinaryOp.constructMRAppendLop(
- getInput().get(_paramIndexMap.get(Statement.GAGG_TARGET)),
- getInput().get(_paramIndexMap.get(Statement.GAGG_GROUPS)),
- DataType.MATRIX, getValueType(), true,
- getInput().get(_paramIndexMap.get(Statement.GAGG_TARGET)));
-
- // add the combine lop to parameter list, with a new name
- // "combinedinput"
- inputlops.put(GroupedAggregate.COMBINEDINPUT, append);
- inputlops.remove(Statement.GAGG_TARGET);
- inputlops.remove(Statement.GAGG_GROUPS);
-
- }
-
- int colwise = -1;
- long outputDim1=-1, outputDim2=-1;
- Lop numGroups = inputlops.get(Statement.GAGG_NUM_GROUPS);
- if ( !dimsKnown() && numGroups != null && numGroups instanceof Data && ((Data)numGroups).isLiteral() ) {
- long ngroups = ((Data)numGroups).getLongValue();
-
- Lop input = inputlops.get(GroupedAggregate.COMBINEDINPUT);
- long inDim1 = input.getOutputParameters().getNumRows();
- long inDim2 = input.getOutputParameters().getNumCols();
- if(inDim1 > 0 && inDim2 > 0 ) {
- if ( inDim1 > inDim2 )
- colwise = 1;
- else
- colwise = 0;
- }
-
- if ( colwise == 1 ) {
- outputDim1 = ngroups;
- outputDim2 = 1;
- }
- else if ( colwise == 0 ) {
- outputDim1 = 1;
- outputDim2 = ngroups;
- }
-
- }
-
- GroupedAggregate grp_agg = new GroupedAggregate(inputlops, getDataType(), getValueType());
-
- // output dimensions are unknown at compilation time
- grp_agg.getOutputParameters().setDimensions(outputDim1, outputDim2, getRowsInBlock(), getColsInBlock(), -1);
- grp_agg.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- setLops(grp_agg);
- setRequiresReblock( true );
- }
- else //CP
- {
- GroupedAggregate grp_agg = new GroupedAggregate(inputlops,
- getDataType(), getValueType(), et);
- // output dimensions are unknown at compilation time
- grp_agg.getOutputParameters().setDimensions(-1, -1, -1, -1, -1);
- grp_agg.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- // introduce a reblock lop only if it is NOT single_node execution
- if( et == ExecType.CP){
- //force blocked output in CP (see below)
- grp_agg.getOutputParameters().setDimensions(-1, 1, getRowsInBlock(), getColsInBlock(), -1);
- }
- else if(et == ExecType.SPARK) {
- grp_agg.getOutputParameters().setDimensions(-1, 1, -1, -1, -1);
- setRequiresReblock( true );
- }
- //grouped agg, w/o reblock in CP
- setLops(grp_agg);
-
- }
- }
-
- /**
- *
- * @param inputlops
- * @param et
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsRemoveEmpty(HashMap<String, Lop> inputlops, ExecType et)
- throws HopsException, LopsException
- {
- Hop targetHop = getInput().get(_paramIndexMap.get("target"));
- Hop marginHop = getInput().get(_paramIndexMap.get("margin"));
- Hop selectHop = (_paramIndexMap.get("select") != null) ? getInput().get(_paramIndexMap.get("select")):null;
-
- if( et == ExecType.CP || et == ExecType.CP_FILE )
- {
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
- setLops(pbilop);
-
- /*DISABLED CP PMM (see for example, MDA Bivar test, requires size propagation on recompile)
- if( et == ExecType.CP && isTargetDiagInput() && marginHop instanceof LiteralOp
- && ((LiteralOp)marginHop).getStringValue().equals("rows")
- && _outputPermutationMatrix ) //SPECIAL CASE SELECTION VECTOR
- {
- //TODO this special case could be taken into account for memory estimates in order
- // to reduce the estimates for the input diag and subsequent matrix multiply
-
- //get input vector (without materializing diag())
- Hop input = targetHop.getInput().get(0);
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- MemoTable memo = new MemoTable();
-
- boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp)input).isPPredOperation());
-
- //step1: compute index vectors
- Hop ppred0 = input;
- if( !isPPredInput ) { //ppred only if required
- ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp("0",0));
- HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen);
- ppred0.refreshSizeInformation();
- ppred0.computeMemEstimate(memo); //select exec type
- HopRewriteUtils.copyLineNumbers(this, ppred0);
- }
-
- UnaryOp cumsum = new UnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, ppred0);
- HopRewriteUtils.setOutputBlocksizes(cumsum, brlen, bclen);
- cumsum.refreshSizeInformation();
- cumsum.computeMemEstimate(memo); //select exec type
- HopRewriteUtils.copyLineNumbers(this, cumsum);
-
- BinaryOp sel = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, ppred0, cumsum);
- HopRewriteUtils.setOutputBlocksizes(sel, brlen, bclen);
- sel.refreshSizeInformation();
- sel.computeMemEstimate(memo); //select exec type
- HopRewriteUtils.copyLineNumbers(this, sel);
- Lop loutput = sel.constructLops();
-
- //Step 4: cleanup hops (allow for garbage collection)
- HopRewriteUtils.removeChildReference(ppred0, input);
-
- setLops( loutput );
- }
- else //GENERAL CASE
- {
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin( et, inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType());
-
- pbilop.getOutputParameters().setDimensions(getDim1(),getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pbilop);
- setLops(pbilop);
- }
- */
- }
- else if( et == ExecType.MR )
- {
- //special compile for mr removeEmpty-diag
- if( isTargetDiagInput() && marginHop instanceof LiteralOp
- && ((LiteralOp)marginHop).getStringValue().equals("rows") )
- {
- //get input vector (without materializing diag())
- Hop input = targetHop.getInput().get(0);
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- MemoTable memo = new MemoTable();
-
- boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp)input).isPPredOperation());
-
- //step1: compute index vectors
- Hop ppred0 = input;
- if( !isPPredInput ) { //ppred only if required
- ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0));
- HopRewriteUtils.updateHopCharacteristics(ppred0, brlen, bclen, memo, this);
- }
-
- UnaryOp cumsum = new UnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, ppred0);
- HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, memo, this);
-
- Lop loutput = null;
- double mest = AggBinaryOp.getMapmmMemEstimate(input.getDim1(), 1, brlen, bclen, -1, brlen, bclen, brlen, bclen, -1, 1, true);
- double mbudget = OptimizerUtils.getRemoteMemBudgetMap(true);
- if( _outputPermutationMatrix && mest < mbudget ) //SPECIAL CASE: SELECTION VECTOR
- {
- BinaryOp sel = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, ppred0, cumsum);
- HopRewriteUtils.updateHopCharacteristics(sel, brlen, bclen, memo, this);
- loutput = sel.constructLops();
- }
- else //GENERAL CASE: GENERAL PERMUTATION MATRIX
- {
- //max ensures non-zero entries and at least one output row
- BinaryOp max = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MAX, cumsum, new LiteralOp(1));
- HopRewriteUtils.updateHopCharacteristics(max, brlen, bclen, memo, this);
-
- DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(input);
- seq.setName("tmp4");
- HopRewriteUtils.updateHopCharacteristics(seq, brlen, bclen, memo, this);
-
- //step 2: compute removeEmpty(rows) output via table, seq guarantees right column dimension
- //note: weights always the input (even if isPPredInput) because input also includes 0s
- TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, max, seq, input);
- HopRewriteUtils.setOutputBlocksizes(table, brlen, bclen);
- table.refreshSizeInformation();
- table.setForcedExecType(ExecType.MR); //force MR
- HopRewriteUtils.copyLineNumbers(this, table);
- table.setDisjointInputs(true);
- table.setOutputEmptyBlocks(_outputEmptyBlocks);
- loutput = table.constructLops();
-
- HopRewriteUtils.removeChildReference(table, input);
- }
-
- //Step 4: cleanup hops (allow for garbage collection)
- HopRewriteUtils.removeChildReference(ppred0, input);
-
- setLops( loutput );
- }
- //default mr remove empty
- else if( et == ExecType.MR )
- {
- //TODO additional physical operator if offsets fit in memory
-
- if( !(marginHop instanceof LiteralOp) )
- throw new HopsException("Parameter 'margin' must be a literal argument.");
-
- Hop input = targetHop;
- long rlen = input.getDim1();
- long clen = input.getDim2();
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- long nnz = input.getNnz();
- boolean rmRows = ((LiteralOp)marginHop).getStringValue().equals("rows");
-
- //construct lops via new partial hop dag and subsequent lops construction
- //in order to reuse of operator selection decisions
-
- BinaryOp ppred0 = null;
- Hop emptyInd = null;
-
- if(selectHop == null) {
- //Step1: compute row/col non-empty indicators
- ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0));
- HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen);
- ppred0.refreshSizeInformation();
- ppred0.setForcedExecType(ExecType.MR); //always MR
- HopRewriteUtils.copyLineNumbers(this, ppred0);
-
- emptyInd = ppred0;
- if( !((rmRows && clen == 1) || (!rmRows && rlen==1)) ){
- emptyInd = new AggUnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, rmRows?Direction.Row:Direction.Col, ppred0);
- HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen);
- emptyInd.refreshSizeInformation();
- emptyInd.setForcedExecType(ExecType.MR); //always MR
- HopRewriteUtils.copyLineNumbers(this, emptyInd);
- }
- } else {
- emptyInd = selectHop;
- HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen);
- emptyInd.refreshSizeInformation();
- emptyInd.setForcedExecType(ExecType.MR); //always MR
- HopRewriteUtils.copyLineNumbers(this, emptyInd);
- }
-
- //Step 2: compute row offsets for non-empty rows
- Hop cumsumInput = emptyInd;
- if( !rmRows ){
- cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
- HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
- }
-
- UnaryOp cumsum = new UnaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, cumsumInput);
- HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
-
- Hop cumsumOutput = cumsum;
- if( !rmRows ){
- cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
- HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
- }
-
- Hop maxDim = new AggUnaryOp("tmp4", DataType.SCALAR, ValueType.DOUBLE, AggOp.MAX, Direction.RowCol, cumsumOutput); //alternative: right indexing
- HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
-
- BinaryOp offsets = new BinaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cumsumOutput, emptyInd);
- HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
-
- //Step 3: gather non-empty rows/cols into final results
- Lop linput = input.constructLops();
- Lop loffset = offsets.constructLops();
- Lop lmaxdim = maxDim.constructLops();
-
- boolean requiresRep = ((clen>bclen || clen<=0) && rmRows)
- || ((rlen>brlen || rlen<=0) && !rmRows);
-
- if( requiresRep ) {
- Lop pos = createOffsetLop(input, rmRows); //ncol of left input (determines num replicates)
- loffset = new RepMat(loffset, pos, rmRows, DataType.MATRIX, ValueType.DOUBLE);
- loffset.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
- setLineNumbers(loffset);
- }
-
- Group group1 = new Group(linput, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group1);
- group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
-
- Group group2 = new Group( loffset, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group2);
- group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
-
- HashMap<String, Lop> inMap = new HashMap<String, Lop>();
- inMap.put("target", group1);
- inMap.put("offset", group2);
- inMap.put("maxdim", lmaxdim);
- inMap.put("margin", inputlops.get("margin"));
-
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
-
- Group group3 = new Group( pbilop, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group3);
- group3.getOutputParameters().setDimensions(-1, -1, brlen, bclen, -1);
-
- Aggregate finalagg = new Aggregate(group3, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), ExecType.MR);
- setOutputDimensions(finalagg);
- setLineNumbers(finalagg);
-
- //Step 4: cleanup hops (allow for garbage collection)
- if(selectHop == null)
- HopRewriteUtils.removeChildReference(ppred0, input);
-
- setLops(finalagg);
- }
- }
- else if( et == ExecType.SPARK )
- {
- if( !(marginHop instanceof LiteralOp) )
- throw new HopsException("Parameter 'margin' must be a literal argument.");
-
- Hop input = targetHop;
- long rlen = input.getDim1();
- long clen = input.getDim2();
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- boolean rmRows = ((LiteralOp)marginHop).getStringValue().equals("rows");
-
- //construct lops via new partial hop dag and subsequent lops construction
- //in order to reuse of operator selection decisions
- BinaryOp ppred0 = null;
- Hop emptyInd = null;
-
- if(selectHop == null) {
- //Step1: compute row/col non-empty indicators
- ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0));
- HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen);
- ppred0.refreshSizeInformation();
- ppred0.setForcedExecType(ExecType.SPARK); //always Spark
- HopRewriteUtils.copyLineNumbers(this, ppred0);
-
- emptyInd = ppred0;
- if( !((rmRows && clen == 1) || (!rmRows && rlen==1)) ){
- emptyInd = new AggUnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, rmRows?Direction.Row:Direction.Col, ppred0);
- HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen);
- emptyInd.refreshSizeInformation();
- emptyInd.setForcedExecType(ExecType.SPARK); //always Spark
- HopRewriteUtils.copyLineNumbers(this, emptyInd);
- }
- } else {
- emptyInd = selectHop;
- HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen);
- emptyInd.refreshSizeInformation();
- emptyInd.setForcedExecType(ExecType.SPARK); //always Spark
- HopRewriteUtils.copyLineNumbers(this, emptyInd);
- }
-
- //Step 2: compute row offsets for non-empty rows
- Hop cumsumInput = emptyInd;
- if( !rmRows ){
- cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
- HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
- }
-
- UnaryOp cumsum = new UnaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, cumsumInput);
- HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
-
- Hop cumsumOutput = cumsum;
- if( !rmRows ){
- cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
- HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
- }
-
- Hop maxDim = new AggUnaryOp("tmp4", DataType.SCALAR, ValueType.DOUBLE, AggOp.MAX, Direction.RowCol, cumsumOutput); //alternative: right indexing
- HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
-
- BinaryOp offsets = new BinaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cumsumOutput, emptyInd);
- HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
-
- //Step 3: gather non-empty rows/cols into final results
- Lop linput = input.constructLops();
- Lop loffset = offsets.constructLops();
- Lop lmaxdim = maxDim.constructLops();
-
- HashMap<String, Lop> inMap = new HashMap<String, Lop>();
- inMap.put("target", linput);
- inMap.put("offset", loffset);
- inMap.put("maxdim", lmaxdim);
- inMap.put("margin", inputlops.get("margin"));
-
- if ( !FORCE_DIST_RM_EMPTY && isRemoveEmptyBcSP())
- _bRmEmptyBC = true;
-
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin( inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et, _bRmEmptyBC);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
-
- //Step 4: cleanup hops (allow for garbage collection)
- if(selectHop == null)
- HopRewriteUtils.removeChildReference(ppred0, input);
-
- setLops(pbilop);
-
- //NOTE: in contrast to mr, replication and aggregation handled instruction-local
- }
- }
-
- /**
- *
- * @param inputlops
- * @param et
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsRExpand(HashMap<String, Lop> inputlops, ExecType et)
- throws HopsException, LopsException
- {
- if( et == ExecType.CP || et == ExecType.SPARK )
- {
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
- setLops(pbilop);
- }
- else if( et == ExecType.MR )
- {
- ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
- setOutputDimensions(pbilop);
- setLineNumbers(pbilop);
-
- Group group1 = new Group( pbilop, Group.OperationTypes.Sort, getDataType(), getValueType());
- setOutputDimensions(group1);
- setLineNumbers(group1);
-
- Aggregate finalagg = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), ExecType.MR);
- setOutputDimensions(finalagg);
- setLineNumbers(finalagg);
- setLops(finalagg);
- }
- }
-
-
- @Override
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" " + _op);
- }
-
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if( _op == ParamBuiltinOp.RMEMPTY )
- {
- Hop marginHop = getInput().get(_paramIndexMap.get("margin"));
- boolean cols = marginHop instanceof LiteralOp
- && "cols".equals(((LiteralOp)marginHop).getStringValue());
-
- //remove empty has additional internal memory requirements for
- //computing selection vectors
- if( cols )
- {
- //selection vector: boolean array in the number of columns
- ret += OptimizerUtils.BIT_SIZE * dim2;
-
- //removeEmpty-cols has additional memory requirements for intermediate
- //data structures in order to make this a cache-friendly operation.
- ret += OptimizerUtils.INT_SIZE * dim2;
- }
- else //rows
- {
- //selection vector: boolean array in the number of rows
- ret += OptimizerUtils.BIT_SIZE * dim1;
- }
- }
- else if( _op == ParamBuiltinOp.REXPAND )
- {
- Hop dir = getInput().get(_paramIndexMap.get("dir"));
- String dirVal = ((LiteralOp)dir).getStringValue();
- if( "rows".equals(dirVal) )
- {
- //rexpand w/ rows direction has additional memory requirements for
- //intermediate data structures in order to prevent performance issues
- //due to random output row access (to make this cache-friendly)
- //NOTE: bounded by blocksize configuration: at most 12MB
- ret = (OptimizerUtils.DOUBLE_SIZE + OptimizerUtils.INT_SIZE)
- * Math.min(dim1, 1024*1024);
- }
- }
-
- return ret;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- //CDF always known because
-
- long[] ret = null;
-
- Hop input = getInput().get(_paramIndexMap.get("target"));
- MatrixCharacteristics mc = memo.getAllInputStats(input);
-
- if( _op == ParamBuiltinOp.GROUPEDAGG )
- {
- // Get the number of groups provided as part of aggregate() invocation, whenever available.
- if ( _paramIndexMap.get(Statement.GAGG_NUM_GROUPS) != null ) {
- Hop ngroups = getInput().get(_paramIndexMap.get(Statement.GAGG_NUM_GROUPS));
- if(ngroups != null && ngroups instanceof LiteralOp) {
- long m = HopRewriteUtils.getIntValueSafe((LiteralOp)ngroups);
- return new long[]{m,1,m};
- }
- }
-
- // Output dimensions are completely data dependent. In the worst case,
- // #groups = #rows in the grouping attribute (e.g., categorical attribute is an ID column, say EmployeeID).
- // In such a case, #rows in the output = #rows in the input. Also, output sparsity is
- // likely to be 1.0 (e.g., groupedAgg(groups=<a ID column>, fn="count"))
- long m = mc.getRows();
- if ( m >= 1 ) {
- ret = new long[]{m, 1, m};
- }
- }
- else if( _op == ParamBuiltinOp.RMEMPTY )
- {
- // similar to groupedagg because in the worst-case ouputsize eq inputsize
- // #nnz is exactly the same as in the input but sparsity can be higher if dimensions.
- // change (denser output).
- if ( mc.dimsKnown() ) {
- String margin = "rows";
- Hop marginHop = getInput().get(_paramIndexMap.get("margin"));
- if( marginHop instanceof LiteralOp
- && "cols".equals(((LiteralOp)marginHop).getStringValue()) )
- margin = new String("cols");
-
- MatrixCharacteristics mcSelect = null;
- if (_paramIndexMap.get("select") != null) {
- Hop select = getInput().get(_paramIndexMap.get("select"));
- mcSelect = memo.getAllInputStats(select);
- }
-
- long lDim1 = 0, lDim2 = 0;
- if( margin.equals("rows") ) {
- lDim1 = (mcSelect == null || !mcSelect.nnzKnown() ) ? mc.getRows(): mcSelect.getNonZeros();
- lDim2 = mc.getCols();
- } else {
- lDim1 = mc.getRows();
- lDim2 = (mcSelect == null || !mcSelect.nnzKnown() ) ? mc.getCols(): mcSelect.getNonZeros();
- }
- ret = new long[]{lDim1, lDim2, mc.getNonZeros()};
- }
- }
- else if( _op == ParamBuiltinOp.REPLACE )
- {
- // the worst-case estimate from the input directly propagates to the output
- // #nnz depends on the replacement pattern and value, same as input if non-zero
- if ( mc.dimsKnown() )
- {
- if( isNonZeroReplaceArguments() )
- ret = new long[]{mc.getRows(), mc.getCols(), mc.getNonZeros()};
- else
- ret = new long[]{mc.getRows(), mc.getCols(), -1};
- }
- }
- else if( _op == ParamBuiltinOp.REXPAND )
- {
- //dimensions are exactly known from input, sparsity unknown but upper bounded by nrow(v)
- //note: cannot infer exact sparsity due to missing cast for outer and potential cutoff for table
- //but very good sparsity estimate possible (number of non-zeros in input)
- Hop max = getInput().get(_paramIndexMap.get("max"));
- Hop dir = getInput().get(_paramIndexMap.get("dir"));
- double maxVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)max);
- String dirVal = ((LiteralOp)dir).getStringValue();
- if( mc.dimsKnown() )
- {
- long lnnz = mc.nnzKnown() ? mc.getNonZeros() : mc.getRows();
- if( "cols".equals(dirVal) ) { //expand horizontally
- ret = new long[]{mc.getRows(), UtilFunctions.toLong(maxVal), lnnz};
- }
- else if( "rows".equals(dirVal) ){ //expand vertically
- ret = new long[]{UtilFunctions.toLong(maxVal), mc.getRows(), lnnz};
- }
- }
- }
-
- return ret;
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return false;
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if( _op == ParamBuiltinOp.TRANSFORM )
- {
- // force remote execution type here.
- // At runtime, cp-side transform is triggered for small files.
- return REMOTE;
- }
-
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if ( _op == ParamBuiltinOp.GROUPEDAGG
- && this.getInput().get(0).areDimsBelowThreshold() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- switch( _op )
- {
- case CDF:
- case INVCDF:
- //do nothing; CDF is a scalar
- break;
-
- case GROUPEDAGG: {
- // output dimension dim1 is completely data dependent
- long ldim1 = -1;
- if ( _paramIndexMap.get(Statement.GAGG_NUM_GROUPS) != null ) {
- Hop ngroups = getInput().get(_paramIndexMap.get(Statement.GAGG_NUM_GROUPS));
- if(ngroups != null && ngroups instanceof LiteralOp) {
- ldim1 = HopRewriteUtils.getIntValueSafe((LiteralOp)ngroups);
- }
- }
-
- setDim1( ldim1 );
- setDim2( 1 );
- break;
- }
- case RMEMPTY: {
- //one output dimension dim1 or dim2 is completely data dependent
- Hop target = getInput().get(_paramIndexMap.get("target"));
- Hop margin = getInput().get(_paramIndexMap.get("margin"));
- if( margin instanceof LiteralOp ) {
- LiteralOp lmargin = (LiteralOp)margin;
- if( "rows".equals(lmargin.getStringValue()) )
- setDim2( target.getDim2() );
- else if( "cols".equals(lmargin.getStringValue()) )
- setDim1( target.getDim1() );
- }
- setNnz( target.getNnz() );
- break;
- }
- case REPLACE: {
- //dimensions are exactly known from input, sparsity might increase/decrease if pattern/replacement 0
- Hop target = getInput().get(_paramIndexMap.get("target"));
- setDim1( target.getDim1() );
- setDim2( target.getDim2() );
- if( isNonZeroReplaceArguments() )
- setNnz( target.getNnz() );
-
- break;
- }
- case REXPAND: {
- //dimensions are exactly known from input, sparsity unknown but upper bounded by nrow(v)
- //note: cannot infer exact sparsity due to missing cast for outer and potential cutoff for table
- Hop target = getInput().get(_paramIndexMap.get("target"));
- Hop max = getInput().get(_paramIndexMap.get("max"));
- Hop dir = getInput().get(_paramIndexMap.get("dir"));
- double maxVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)max);
- String dirVal = ((LiteralOp)dir).getStringValue();
-
- if( "cols".equals(dirVal) ) { //expand horizontally
- setDim1(target.getDim1());
- setDim2(UtilFunctions.toLong(maxVal));
- }
- else if( "rows".equals(dirVal) ){ //expand vertically
- setDim1(UtilFunctions.toLong(maxVal));
- setDim2(target.getDim1());
- }
-
- break;
- }
- default:
- //do nothing
- break;
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object clone() throws CloneNotSupportedException
- {
- ParameterizedBuiltinOp ret = new ParameterizedBuiltinOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._outputEmptyBlocks = _outputEmptyBlocks;
- ret._outputPermutationMatrix = _outputPermutationMatrix;
- ret._paramIndexMap = (HashMap<String, Integer>) _paramIndexMap.clone();
- //note: no deep cp of params since read-only
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof ParameterizedBuiltinOp) )
- return false;
-
- ParameterizedBuiltinOp that2 = (ParameterizedBuiltinOp)that;
- boolean ret = (_op == that2._op
- && _paramIndexMap!=null && that2._paramIndexMap!=null
- && _paramIndexMap.size() == that2._paramIndexMap.size()
- && _outputEmptyBlocks == that2._outputEmptyBlocks
- && _outputPermutationMatrix == that2._outputPermutationMatrix );
- if( ret )
- {
- for( Entry<String,Integer> e : _paramIndexMap.entrySet() )
- {
- String key1 = e.getKey();
- int pos1 = e.getValue();
- int pos2 = that2._paramIndexMap.get(key1);
- ret &= ( that2.getInput().get(pos2)!=null
- && getInput().get(pos1) == that2.getInput().get(pos2) );
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- @Override
- public boolean isTransposeSafe()
- {
- boolean ret = false;
-
- try
- {
- if( _op == ParamBuiltinOp.GROUPEDAGG )
- {
- int ix = _paramIndexMap.get(Statement.GAGG_FN);
- Hop fnHop = getInput().get(ix);
- ret = (fnHop instanceof LiteralOp && Statement.GAGG_FN_SUM.equals(((LiteralOp)fnHop).getStringValue()) );
- }
- }
- catch(Exception ex) {
- //silent false
- LOG.warn("Check for transpose-safeness failed, continue assuming false.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public boolean isCountFunction()
- {
- boolean ret = false;
-
- try
- {
- if( _op == ParamBuiltinOp.GROUPEDAGG )
- {
- int ix = _paramIndexMap.get(Statement.GAGG_FN);
- Hop fnHop = getInput().get(ix);
- ret = (fnHop instanceof LiteralOp && Statement.GAGG_FN_COUNT.equals(((LiteralOp)fnHop).getStringValue()) );
- }
- }
- catch(Exception ex){
- //silent false
- LOG.warn("Check for count function failed, continue assuming false.", ex);
- }
-
- return ret;
- }
-
- /**
- * Only applies to REPLACE.
- * @return
- */
- private boolean isNonZeroReplaceArguments()
- {
- boolean ret = false;
- try
- {
- Hop pattern = getInput().get(_paramIndexMap.get("pattern"));
- Hop replace = getInput().get(_paramIndexMap.get("replacement"));
- if( pattern instanceof LiteralOp && ((LiteralOp)pattern).getDoubleValue()!=0d &&
- replace instanceof LiteralOp && ((LiteralOp)replace).getDoubleValue()!=0d )
- {
- ret = true;
- }
- }
- catch(Exception ex)
- {
- LOG.warn(ex.getMessage());
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public boolean isTargetDiagInput()
- {
- Hop targetHop = getTargetHop();
-
- //input vector (guarantees diagV2M), implies remove rows
- return ( targetHop instanceof ReorgOp
- && ((ReorgOp)targetHop).getOp()==ReOrgOp.DIAG
- && targetHop.getInput().get(0).getDim2() == 1 );
- }
-
- /**
- * This will check if there is sufficient memory locally (twice the size of second matrix, for original and sort data), and remotely (size of second matrix (sorted data)).
- * @return
- */
- private boolean isRemoveEmptyBcSP() // TODO find if 2 x size needed.
- {
- boolean ret = false;
- Hop input = getInput().get(0);
-
- //note: both cases (partitioned matrix, and sorted double array), require to
- //fit the broadcast twice into the local memory budget. Also, the memory
- //constraint only needs to take the rhs into account because the output is
- //guaranteed to be an aggregate of <=16KB
-
- double size = input.dimsKnown() ?
- OptimizerUtils.estimateSize(input.getDim1(), 1) : //dims known and estimate fits
- input.getOutputMemEstimate(); //dims unknown but worst-case estimate fits
-
- if( OptimizerUtils.checkSparkBroadcastMemoryBudget(size) ) {
- ret = true;
- }
-
- return ret;
- }
-
-}
[05/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ProgramConverter.java
deleted file mode 100644
index 1077d3e..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ProgramConverter.java
+++ /dev/null
@@ -1,2368 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.StringTokenizer;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobConf;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-
-/**
- * Static functionalities for
- * * creating deep copies of program blocks, instructions, function program blocks
- * * serializing and parsing of programs, program blocks, functions program blocks
- *
- * TODO: CV, EL, ELUse program blocks not considered so far (not for BI 2.0 release)
- * TODO: rewrite class to instance-based invocation (grown gradually and now inappropriate design)
- *
- */
-public class ProgramConverter
-{
-
- protected static final Log LOG = LogFactory.getLog(ProgramConverter.class.getName());
-
- //use escaped unicodes for separators in order to prevent string conflict
- public static final String NEWLINE = "\n"; //System.lineSeparator();
- public static final String COMPONENTS_DELIM = "\u236e"; //semicolon w/ bar; ";";
- public static final String ELEMENT_DELIM = "\u236a"; //comma w/ bar; ",";
- public static final String DATA_FIELD_DELIM = "\u007c"; //"|";
- public static final String KEY_VALUE_DELIM = "\u003d"; //"=";
- public static final String LEVELIN = "\u23a8"; //variant of left curly bracket; "\u007b"; //"{";
- public static final String LEVELOUT = "\u23ac"; //variant of right curly bracket; "\u007d"; //"}";
- public static final String EMPTY = "null";
- public static final String EXT_FUNCTION = "extfunct";
-
- //public static final String CP_ROOT_THREAD_SEPARATOR = "/";//File.separator;
- public static final String CP_ROOT_THREAD_ID = "_t0";
- public static final String CP_CHILD_THREAD = "_t";
-
- public static final String PARFOR_CDATA_BEGIN = "<![CDATA[";
- public static final String PARFOR_CDATA_END = " ]]>";
-
- public static final String PARFOR_PROG_BEGIN = " PROG" + LEVELIN;
- public static final String PARFOR_PROG_END = LEVELOUT;
- public static final String PARFORBODY_BEGIN = PARFOR_CDATA_BEGIN+"PARFORBODY" + LEVELIN;
- public static final String PARFORBODY_END = LEVELOUT+PARFOR_CDATA_END;
- public static final String PARFOR_VARS_BEGIN = "VARS: ";
- public static final String PARFOR_VARS_END = "";
- public static final String PARFOR_PBS_BEGIN = " PBS" + LEVELIN;
- public static final String PARFOR_PBS_END = LEVELOUT;
- public static final String PARFOR_INST_BEGIN = " INST: ";
- public static final String PARFOR_INST_END = "";
- public static final String PARFOR_EC_BEGIN = " EC: ";
- public static final String PARFOR_EC_END = "";
- public static final String PARFOR_PB_BEGIN = " PB" + LEVELIN;
- public static final String PARFOR_PB_END = LEVELOUT;
- public static final String PARFOR_PB_WHILE = " WHILE" + LEVELIN;
- public static final String PARFOR_PB_FOR = " FOR" + LEVELIN;
- public static final String PARFOR_PB_PARFOR = " PARFOR" + LEVELIN;
- public static final String PARFOR_PB_IF = " IF" + LEVELIN;
- public static final String PARFOR_PB_FC = " FC" + LEVELIN;
- public static final String PARFOR_PB_EFC = " EFC" + LEVELIN;
-
- public static final String PARFOR_CONF_STATS = "stats";
-
-
- //exception msgs
- public static final String NOT_SUPPORTED_EXTERNALFUNCTION_PB = "Not supported: ExternalFunctionProgramBlock contains MR instructions. " +
- "(ExternalFunctionPRogramBlockCP can be used)";
- public static final String NOT_SUPPORTED_MR_INSTRUCTION = "Not supported: Instructions of type other than CP instructions";
- public static final String NOT_SUPPORTED_MR_PARFOR = "Not supported: Nested ParFOR REMOTE_MR due to possible deadlocks." +
- "(LOCAL can be used for innner ParFOR)";
- public static final String NOT_SUPPORTED_PB = "Not supported: type of program block";
- public static final String NOT_SUPPORTED_EXECUTION_CONTEXT = "Parsing of external system execution context not supported yet.";
-
-
- ////////////////////////////////
- // CREATION of DEEP COPIES
- ////////////////////////////////
-
- /**
- * Creates a deep copy of the given execution context.
- * For rt_platform=Hadoop, execution context has a symbol table.
- *
- * @throws CloneNotSupportedException
- * @throws DMLRuntimeException
- */
- public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec)
- throws CloneNotSupportedException, DMLRuntimeException
- {
- ExecutionContext cpec = ExecutionContextFactory.createContext(false, ec.getProgram());
- cpec.setVariables((LocalVariableMap) ec.getVariables().clone());
-
- //handle result variables with in-place update flag
- //(each worker requires its own copy of the empty matrix object)
- for( String var : cpec.getVariables().keySet() ) {
- Data dat = cpec.getVariables().get(var);
- if( dat instanceof MatrixObject && ((MatrixObject)dat).isUpdateInPlaceEnabled() ) {
- MatrixObject mo = (MatrixObject)dat;
- MatrixObject moNew = new MatrixObject(mo);
- if( mo.getNnz() != 0 ){
- // If output matrix is not empty (NNZ != 0), then local copy is created so that
- // update in place operation can be applied.
- MatrixBlock mbVar = mo.acquireRead();
- moNew.acquireModify (new MatrixBlock(mbVar));
- mo.release();
- } else {
- //create empty matrix block w/ dense representation (preferred for update in-place)
- //Creating a dense matrix block is valid because empty block not allocated and transfer
- // to sparse representation happens in left indexing in place operation.
- moNew.acquireModify(new MatrixBlock((int)mo.getNumRows(), (int)mo.getNumColumns(), false));
- }
- moNew.release();
- cpec.setVariable(var, moNew);
- }
- }
-
- return cpec;
- }
-
- /**
- * This recursively creates a deep copy of program blocks and transparently replaces filenames according to the
- * specified parallel worker in order to avoid conflicts between parworkers. This happens recursively in order
- * to support arbitrary control-flow constructs within a parfor.
- *
- * @param childBlocks
- * @param pid
- * @param plain full deep copy without id replacement
- * @param forceDeepCopy
- *
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ArrayList<ProgramBlock> rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks, long pid, int IDPrefix, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<ProgramBlock> tmp = new ArrayList<ProgramBlock>();
-
- for( ProgramBlock pb : childBlocks )
- {
- Program prog = pb.getProgram();
- ProgramBlock tmpPB = null;
-
- if( pb instanceof WhileProgramBlock )
- {
- tmpPB = createDeepCopyWhileProgramBlock((WhileProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
- }
- else if( pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock) )
- {
- tmpPB = createDeepCopyForProgramBlock((ForProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy );
- }
- else if( pb instanceof ParForProgramBlock )
- {
- ParForProgramBlock pfpb = (ParForProgramBlock) pb;
- if( ParForProgramBlock.ALLOW_NESTED_PARALLELISM )
- tmpPB = createDeepCopyParForProgramBlock(pfpb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
- else
- tmpPB = createDeepCopyForProgramBlock((ForProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
- }
- else if( pb instanceof IfProgramBlock )
- {
- tmpPB = createDeepCopyIfProgramBlock((IfProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
- }
- else //last-level program block
- {
- tmpPB = new ProgramBlock(prog); // general case use for most PBs
-
- //for recompile in the master node JVM
- tmpPB.setStatementBlock(createStatementBlockCopy(pb.getStatementBlock(), pid, plain, forceDeepCopy));
- //tmpPB.setStatementBlock(pb.getStatementBlock());
- tmpPB.setThreadID(pid);
- }
-
- //copy instructions
- tmpPB.setInstructions( createDeepCopyInstructionSet(pb.getInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
-
- //copy symbol table
- //tmpPB.setVariables( pb.getVariables() ); //implicit cloning
-
- tmp.add(tmpPB);
- }
-
- return tmp;
- }
-
- /**
- *
- * @param wpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static WhileProgramBlock createDeepCopyWhileProgramBlock(WhileProgramBlock wpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> predinst = createDeepCopyInstructionSet(wpb.getPredicate(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true);
- WhileProgramBlock tmpPB = new WhileProgramBlock(prog, predinst);
- tmpPB.setPredicateResultVar( wpb.getPredicateResultVar() );
- tmpPB.setStatementBlock( createWhileStatementBlockCopy((WhileStatementBlock) wpb.getStatementBlock(), pid, plain, forceDeepCopy) );
- tmpPB.setThreadID(pid);
-
- tmpPB.setExitInstructions2( createDeepCopyInstructionSet(wpb.getExitInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true));
- tmpPB.setChildBlocks(rcreateDeepCopyProgramBlocks(wpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
-
- return tmpPB;
- }
-
- /**
- *
- * @param ipb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static IfProgramBlock createDeepCopyIfProgramBlock(IfProgramBlock ipb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> predinst = createDeepCopyInstructionSet(ipb.getPredicate(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true);
- IfProgramBlock tmpPB = new IfProgramBlock(prog, predinst);
- tmpPB.setPredicateResultVar( ipb.getPredicateResultVar() );
- tmpPB.setStatementBlock( createIfStatementBlockCopy((IfStatementBlock)ipb.getStatementBlock(), pid, plain, forceDeepCopy ) );
- tmpPB.setThreadID(pid);
-
- tmpPB.setExitInstructions2( createDeepCopyInstructionSet(ipb.getExitInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true));
- tmpPB.setChildBlocksIfBody(rcreateDeepCopyProgramBlocks(ipb.getChildBlocksIfBody(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
- tmpPB.setChildBlocksElseBody(rcreateDeepCopyProgramBlocks(ipb.getChildBlocksElseBody(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
-
- return tmpPB;
- }
-
- /**
- *
- * @param fpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ForProgramBlock createDeepCopyForProgramBlock(ForProgramBlock fpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterablePredicateVars());
- tmpPB.setStatementBlock( createForStatementBlockCopy((ForStatementBlock)fpb.getStatementBlock(), pid, plain, forceDeepCopy));
- tmpPB.setThreadID(pid);
-
- tmpPB.setFromInstructions( createDeepCopyInstructionSet(fpb.getFromInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setToInstructions( createDeepCopyInstructionSet(fpb.getToInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setIncrementInstructions( createDeepCopyInstructionSet(fpb.getIncrementInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setExitInstructions( createDeepCopyInstructionSet(fpb.getExitInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setChildBlocks( rcreateDeepCopyProgramBlocks(fpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy) );
-
- return tmpPB;
- }
-
- /**
- *
- * @param fpb
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ForProgramBlock createShallowCopyForProgramBlock(ForProgramBlock fpb, Program prog )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterablePredicateVars());
-
- tmpPB.setFromInstructions( fpb.getFromInstructions() );
- tmpPB.setToInstructions( fpb.getToInstructions() );
- tmpPB.setIncrementInstructions( fpb.getIncrementInstructions() );
- tmpPB.setExitInstructions( fpb.getExitInstructions() );
- tmpPB.setChildBlocks( fpb.getChildBlocks() );
-
- return tmpPB;
- }
-
- /**
- *
- * @param pfpb
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ForProgramBlock createShallowCopyParForProgramBlock(ParForProgramBlock pfpb, Program prog )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ParForProgramBlock tmpPB = new ParForProgramBlock(prog,pfpb.getIterablePredicateVars(),pfpb.getParForParams());
-
- tmpPB.setStatementBlock( pfpb.getStatementBlock() );
- tmpPB.setResultVariables( pfpb.getResultVariables() );
-
- tmpPB.setFromInstructions( pfpb.getFromInstructions() );
- tmpPB.setToInstructions( pfpb.getToInstructions() );
- tmpPB.setIncrementInstructions( pfpb.getIncrementInstructions() );
- tmpPB.setExitInstructions( pfpb.getExitInstructions() );
- tmpPB.setChildBlocks( pfpb.getChildBlocks() );
-
- return tmpPB;
- }
-
- /**
- *
- * @param pfpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ParForProgramBlock createDeepCopyParForProgramBlock(ParForProgramBlock pfpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ParForProgramBlock tmpPB = null;
-
- if( IDPrefix == -1 ) //still on master node
- tmpPB = new ParForProgramBlock(prog,pfpb.getIterablePredicateVars(),pfpb.getParForParams());
- else //child of remote ParWorker at any level
- tmpPB = new ParForProgramBlock(IDPrefix, prog, pfpb.getIterablePredicateVars(),pfpb.getParForParams());
-
- tmpPB.setStatementBlock( createForStatementBlockCopy( (ForStatementBlock) pfpb.getStatementBlock(), pid, plain, forceDeepCopy) );
- tmpPB.setThreadID(pid);
-
- tmpPB.disableOptimization(); //already done in top-level parfor
- tmpPB.disableMonitorReport(); //already done in top-level parfor
- tmpPB.setResultVariables( pfpb.getResultVariables() );
-
- tmpPB.setFromInstructions( createDeepCopyInstructionSet(pfpb.getFromInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setToInstructions( createDeepCopyInstructionSet(pfpb.getToInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setIncrementInstructions( createDeepCopyInstructionSet(pfpb.getIncrementInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
- tmpPB.setExitInstructions( createDeepCopyInstructionSet(pfpb.getExitInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
-
- //NOTE: Normally, no recursive copy because (1) copied on each execution in this PB anyway
- //and (2) leave placeholders as they are. However, if plain, an explicit deep copy is requested.
- if( plain || forceDeepCopy )
- tmpPB.setChildBlocks( rcreateDeepCopyProgramBlocks(pfpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy) );
- else
- tmpPB.setChildBlocks( pfpb.getChildBlocks() );
-
- return tmpPB;
- }
-
- /**
- * This creates a deep copy of a function program block. The central reference to singletons of function program blocks
- * poses the need for explicit copies in order to prevent conflicting writes of temporary variables (see ExternalFunctionProgramBlock.
- *
- * @param oldName
- * @param pid
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static void createDeepCopyFunctionProgramBlock(String namespace, String oldName, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //fpb guaranteed to be non-null (checked inside getFunctionProgramBlock)
- FunctionProgramBlock fpb = prog.getFunctionProgramBlock(namespace, oldName);
- String fnameNew = (plain)? oldName :(oldName+CP_CHILD_THREAD+pid);
- String fnameNewKey = DMLProgram.constructFunctionKey(namespace,fnameNew);
-
- if( prog.getFunctionProgramBlocks().containsKey(fnameNewKey) )
- return; //prevent redundant deep copy if already existent
-
- //create deep copy
- FunctionProgramBlock copy = null;
- ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>();
- if( fpb.getInputParams()!= null )
- tmp1.addAll(fpb.getInputParams());
- if( fpb.getOutputParams()!= null )
- tmp2.addAll(fpb.getOutputParams());
-
- if( fpb instanceof ExternalFunctionProgramBlockCP )
- {
- ExternalFunctionProgramBlockCP efpb = (ExternalFunctionProgramBlockCP) fpb;
- HashMap<String,String> tmp3 = efpb.getOtherParams();
- if( IDPrefix!=-1 )
- copy = new ExternalFunctionProgramBlockCP(prog,tmp1,tmp2,tmp3,saveReplaceFilenameThreadID(efpb.getBaseDir(),CP_CHILD_THREAD+IDPrefix,CP_CHILD_THREAD+pid));
- else
- copy = new ExternalFunctionProgramBlockCP(prog,tmp1,tmp2,tmp3,saveReplaceFilenameThreadID(efpb.getBaseDir(),CP_ROOT_THREAD_ID,CP_CHILD_THREAD+pid));
- }
- else if( fpb instanceof ExternalFunctionProgramBlock )
- {
- ExternalFunctionProgramBlock efpb = (ExternalFunctionProgramBlock) fpb;
- HashMap<String,String> tmp3 = efpb.getOtherParams();
- if( IDPrefix!=-1 )
- copy = new ExternalFunctionProgramBlock(prog,tmp1,tmp2,tmp3,saveReplaceFilenameThreadID(efpb.getBaseDir(),CP_CHILD_THREAD+IDPrefix, CP_CHILD_THREAD+pid));
- else
- copy = new ExternalFunctionProgramBlock(prog,tmp1,tmp2,tmp3,saveReplaceFilenameThreadID(efpb.getBaseDir(),CP_ROOT_THREAD_ID, CP_CHILD_THREAD+pid));
- }
- else
- {
- if( !fnStack.contains(fnameNewKey) ) {
- fnStack.add(fnameNewKey);
- copy = new FunctionProgramBlock(prog, tmp1, tmp2);
- copy.setChildBlocks( rcreateDeepCopyProgramBlocks(fpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, fpb.isRecompileOnce()) );
- copy.setRecompileOnce( fpb.isRecompileOnce() );
- copy.setThreadID(pid);
- fnStack.remove(fnameNewKey);
- }
- else //stop deep copy for recursive function calls
- copy = fpb;
- }
-
- //copy.setVariables( (LocalVariableMap) fpb.getVariables() ); //implicit cloning
- //note: instructions not used by function program block
-
- //put
- prog.addFunctionProgramBlock(namespace, fnameNew, copy);
- fnCreated.add(DMLProgram.constructFunctionKey(namespace, fnameNew));
- }
-
- /**
- *
- * @param fpb
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static FunctionProgramBlock createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb, HashSet<String> fnStack, HashSet<String> fnCreated)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- if( fpb == null )
- throw new DMLRuntimeException("Unable to create a deep copy of a non-existing FunctionProgramBlock.");
-
- //create deep copy
- FunctionProgramBlock copy = null;
- ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>();
- if( fpb.getInputParams()!= null )
- tmp1.addAll(fpb.getInputParams());
- if( fpb.getOutputParams()!= null )
- tmp2.addAll(fpb.getOutputParams());
-
- copy = new FunctionProgramBlock(fpb.getProgram(), tmp1, tmp2);
- copy.setChildBlocks( rcreateDeepCopyProgramBlocks(fpb.getChildBlocks(), 0, -1, fnStack, fnCreated, true, fpb.isRecompileOnce()) );
- copy.setStatementBlock( fpb.getStatementBlock() );
- copy.setRecompileOnce(fpb.isRecompileOnce());
- //copy.setVariables( (LocalVariableMap) fpb.getVariables() ); //implicit cloning
- //note: instructions not used by function program block
-
- return copy;
- }
-
-
- /**
- * Creates a deep copy of an array of instructions and replaces the placeholders of parworker
- * IDs with the concrete IDs of this parfor instance. This is a helper method uses for generating
- * deep copies of program blocks.
- *
- * @param instSet
- * @param pid
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ArrayList<Instruction> createDeepCopyInstructionSet(ArrayList<Instruction> instSet, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean cpFunctions)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> tmp = new ArrayList<Instruction>();
- for( Instruction inst : instSet )
- {
- if( inst instanceof FunctionCallCPInstruction && cpFunctions )
- {
- FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
- createDeepCopyFunctionProgramBlock( finst.getNamespace(),
- finst.getFunctionName(),
- pid, IDPrefix, prog, fnStack, fnCreated, plain );
- }
-
- tmp.add( cloneInstruction( inst, pid, plain, cpFunctions ) );
- }
-
- return tmp;
- }
-
-
- /**
- *
- * @param pid
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static Instruction cloneInstruction( Instruction oInst, long pid, boolean plain, boolean cpFunctions )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- Instruction inst = null;
- String tmpString = oInst.toString();
-
- try
- {
- if( oInst instanceof CPInstruction || oInst instanceof SPInstruction || oInst instanceof MRInstruction )
- {
- if( oInst instanceof FunctionCallCPInstruction && cpFunctions )
- {
- FunctionCallCPInstruction tmp = (FunctionCallCPInstruction) oInst;
- if( !plain )
- {
- //safe replacement because target variables might include the function name
- //note: this is no update-in-place in order to keep the original function name as basis
- tmpString = tmp.updateInstStringFunctionName(tmp.getFunctionName(), tmp.getFunctionName() + CP_CHILD_THREAD+pid);
- }
- //otherwise: preserve function name
- }
-
- inst = InstructionParser.parseSingleInstruction(tmpString);
- }
- else if( oInst instanceof MRJobInstruction )
- {
- //clone via copy constructor
- inst = new MRJobInstruction( (MRJobInstruction)oInst );
- }
- else
- throw new DMLUnsupportedOperationException("Failed to clone instruction: "+oInst);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //save replacement of thread id references in instructions
- inst = saveReplaceThreadID( inst, ProgramConverter.CP_ROOT_THREAD_ID,
- ProgramConverter.CP_CHILD_THREAD+pid);
-
- return inst;
- }
-
- /**
- *
- * @param sb
- * @return
- * @throws DMLRuntimeException
- */
- public static StatementBlock createStatementBlockCopy( StatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
- throws DMLRuntimeException
- {
- StatementBlock ret = null;
-
- try
- {
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
- && sb != null //forced deep copy for function recompilation
- && (Recompiler.requiresRecompilation( sb.get_hops() ) || forceDeepCopy) )
- {
- //create new statement (shallow copy livein/liveout for recompile, line numbers for explain)
- ret = new StatementBlock();
- ret.setDMLProg(sb.getDMLProg());
- ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- ret.setLiveIn( sb.liveIn() );
- ret.setLiveOut( sb.liveOut() );
- ret.setUpdatedVariables( sb.variablesUpdated() );
- ret.setReadVariables( sb.variablesRead() );
-
- //deep copy hops dag for concurrent recompile
- ArrayList<Hop> hops = Recompiler.deepCopyHopsDag( sb.get_hops() );
- if( !plain )
- Recompiler.updateFunctionNames( hops, pid );
- ret.set_hops( hops );
- ret.updateRecompilationFlag();
- }
- else
- {
- ret = sb;
- }
- }
- catch( Exception ex )
- {
- throw new DMLRuntimeException( ex );
- }
-
- return ret;
- }
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
- public static IfStatementBlock createIfStatementBlockCopy( IfStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
- throws DMLRuntimeException
- {
- IfStatementBlock ret = null;
-
- try
- {
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
- && sb != null //forced deep copy for function recompile
- && (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
- {
- //create new statement (shallow copy livein/liveout for recompile, line numbers for explain)
- ret = new IfStatementBlock();
- ret.setDMLProg(sb.getDMLProg());
- ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- ret.setLiveIn( sb.liveIn() );
- ret.setLiveOut( sb.liveOut() );
- ret.setUpdatedVariables( sb.variablesUpdated() );
- ret.setReadVariables( sb.variablesRead() );
-
- //shallow copy child statements
- ret.setStatements( sb.getStatements() );
-
- //deep copy predicate hops dag for concurrent recompile
- Hop hops = Recompiler.deepCopyHopsDag( sb.getPredicateHops() );
- ret.setPredicateHops( hops );
- ret.updatePredicateRecompilationFlag();
- }
- else
- {
- ret = sb;
- }
- }
- catch( Exception ex )
- {
- throw new DMLRuntimeException( ex );
- }
-
- return ret;
- }
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
- public static WhileStatementBlock createWhileStatementBlockCopy( WhileStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
- throws DMLRuntimeException
- {
- WhileStatementBlock ret = null;
-
- try
- {
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
- && sb != null //forced deep copy for function recompile
- && (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
- {
- //create new statement (shallow copy livein/liveout for recompile, line numbers for explain)
- ret = new WhileStatementBlock();
- ret.setDMLProg(sb.getDMLProg());
- ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- ret.setLiveIn( sb.liveIn() );
- ret.setLiveOut( sb.liveOut() );
- ret.setUpdatedVariables( sb.variablesUpdated() );
- ret.setReadVariables( sb.variablesRead() );
-
- //shallow copy child statements
- ret.setStatements( sb.getStatements() );
-
- //deep copy predicate hops dag for concurrent recompile
- Hop hops = Recompiler.deepCopyHopsDag( sb.getPredicateHops() );
- ret.setPredicateHops( hops );
- ret.updatePredicateRecompilationFlag();
- }
- else
- {
- ret = sb;
- }
- }
- catch( Exception ex )
- {
- throw new DMLRuntimeException( ex );
- }
-
- return ret;
- }
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
- public static ForStatementBlock createForStatementBlockCopy( ForStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
- throws DMLRuntimeException
- {
- ForStatementBlock ret = null;
-
- try
- {
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
- && sb != null
- && ( Recompiler.requiresRecompilation(sb.getFromHops()) ||
- Recompiler.requiresRecompilation(sb.getToHops()) ||
- Recompiler.requiresRecompilation(sb.getIncrementHops()) ||
- forceDeepCopy ) )
- {
- ret = (sb instanceof ParForStatementBlock) ? new ParForStatementBlock() : new ForStatementBlock();
-
- //create new statement (shallow copy livein/liveout for recompile, line numbers for explain)
- ret.setDMLProg(sb.getDMLProg());
- ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- ret.setLiveIn( sb.liveIn() );
- ret.setLiveOut( sb.liveOut() );
- ret.setUpdatedVariables( sb.variablesUpdated() );
- ret.setReadVariables( sb.variablesRead() );
-
- //shallow copy child statements
- ret.setStatements( sb.getStatements() );
-
- //deep copy predicate hops dag for concurrent recompile
- if( sb.requiresFromRecompilation() ){
- Hop hops = Recompiler.deepCopyHopsDag( sb.getFromHops() );
- ret.setFromHops( hops );
- }
- if( sb.requiresToRecompilation() ){
- Hop hops = Recompiler.deepCopyHopsDag( sb.getToHops() );
- ret.setToHops( hops );
- }
- if( sb.requiresIncrementRecompilation() ){
- Hop hops = Recompiler.deepCopyHopsDag( sb.getIncrementHops() );
- ret.setIncrementHops( hops );
- }
- ret.updatePredicateRecompilationFlags();
- }
- else
- {
- ret = sb;
- }
- }
- catch( Exception ex )
- {
- throw new DMLRuntimeException( ex );
- }
-
- return ret;
- }
-
-
- ////////////////////////////////
- // SERIALIZATION
- ////////////////////////////////
-
- /**
- * @param body
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static String serializeParForBody( ParForBody body )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<ProgramBlock> pbs = body.getChildBlocks();
- ArrayList<String> rVnames = body.getResultVarNames();
- ExecutionContext ec = body.getEc();
-
- if( pbs.isEmpty() )
- return PARFORBODY_BEGIN + PARFORBODY_END;
-
- Program prog = pbs.get( 0 ).getProgram();
-
- StringBuilder sb = new StringBuilder();
- sb.append( PARFORBODY_BEGIN );
- sb.append( NEWLINE );
-
- //handle DMLScript UUID (propagate original uuid for writing to scratch space)
- sb.append( DMLScript.getUUID() );
- sb.append( COMPONENTS_DELIM );
- sb.append( NEWLINE );
-
- //handle DML config
- sb.append( ConfigurationManager.getConfig().serializeDMLConfig() );
- sb.append( COMPONENTS_DELIM );
- sb.append( NEWLINE );
-
- //handle additional configurations
- sb.append( PARFOR_CONF_STATS + "=" + DMLScript.STATISTICS );
- sb.append( COMPONENTS_DELIM );
- sb.append( NEWLINE );
-
- //handle program
- sb.append( PARFOR_PROG_BEGIN );
- sb.append( NEWLINE );
- sb.append( serializeProgram(prog, pbs) );
- sb.append( PARFOR_PROG_END );
- sb.append( NEWLINE );
- sb.append( COMPONENTS_DELIM );
- sb.append( NEWLINE );
-
- //handle result variable names
- sb.append( serializeStringArrayList( rVnames ) );
- sb.append( COMPONENTS_DELIM );
-
- //handle execution context
- //note: this includes also the symbol table (serialize only the top-level variable map,
- // (symbol tables for nested/child blocks are created at parse time, on the remote side)
- sb.append( PARFOR_EC_BEGIN );
- sb.append( serializeExecutionContext(ec) );
- sb.append( PARFOR_EC_END );
- sb.append( NEWLINE );
- sb.append( COMPONENTS_DELIM );
- sb.append( NEWLINE );
-
- //handle program blocks -- ONLY instructions, not variables.
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( NEWLINE );
- sb.append( rSerializeProgramBlocks(pbs) );
- sb.append( PARFOR_PBS_END );
- sb.append( NEWLINE );
-
- sb.append( PARFORBODY_END );
-
- return sb.toString();
- }
-
- /**
- *
- * @param prog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static String serializeProgram( Program prog, ArrayList<ProgramBlock> pbs )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //note program contains variables, programblocks and function program blocks
- //but in order to avoid redundancy, we only serialize function program blocks
-
- HashMap<String, FunctionProgramBlock> fpb = prog.getFunctionProgramBlocks();
- HashSet<String> cand = new HashSet<String>();
- rFindSerializationCandidates(pbs, cand);
-
- return rSerializeFunctionProgramBlocks( fpb, cand );
- }
-
- /**
- *
- * @param pbs
- * @param cand
- * @throws DMLRuntimeException
- */
- public static void rFindSerializationCandidates( ArrayList<ProgramBlock> pbs, HashSet<String> cand )
- throws DMLRuntimeException
- {
- for( ProgramBlock pb : pbs )
- {
- if( pb instanceof WhileProgramBlock )
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- rFindSerializationCandidates(wpb.getChildBlocks(), cand );
- }
- else if ( pb instanceof ForProgramBlock || pb instanceof ParForProgramBlock )
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- rFindSerializationCandidates(fpb.getChildBlocks(), cand);
- }
- else if ( pb instanceof IfProgramBlock )
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- rFindSerializationCandidates(ipb.getChildBlocksIfBody(), cand);
- if( ipb.getChildBlocksElseBody() != null )
- rFindSerializationCandidates(ipb.getChildBlocksElseBody(), cand);
- }
- else //all generic program blocks
- {
- for( Instruction inst : pb.getInstructions() )
- if( inst instanceof FunctionCallCPInstruction )
- {
- FunctionCallCPInstruction fci = (FunctionCallCPInstruction) inst;
- String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(), fci.getFunctionName());
- if( !cand.contains(fkey) ) //memoization for multiple calls, recursion
- {
- cand.add( fkey ); //add to candidates
-
- //investigate chains of function calls
- FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fci.getNamespace(), fci.getFunctionName());
- rFindSerializationCandidates(fpb.getChildBlocks(), cand);
- }
- }
- }
- }
- }
-
-
- /**
- *
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- public static String serializeVariables (LocalVariableMap vars)
- throws DMLRuntimeException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( PARFOR_VARS_BEGIN );
- sb.append( vars.serialize() );
- sb.append( PARFOR_VARS_END );
-
- return sb.toString();
- }
-
- /**
- *
- * @param key
- * @param dat
- * @return
- * @throws DMLRuntimeException
- */
- public static String serializeDataObject(String key, Data dat)
- throws DMLRuntimeException
- {
- // SCHEMA: <name>|<datatype>|<valuetype>|value
- // (scalars are serialize by value, matrices by filename)
-
- StringBuilder sb = new StringBuilder();
-
- //prepare data for serialization
- String name = key;
- DataType datatype = dat.getDataType();
- ValueType valuetype = dat.getValueType();
- String value = null;
- String[] matrixMetaData = null;
-
- switch( datatype )
- {
- case SCALAR:
- ScalarObject so = (ScalarObject) dat;
- //name = so.getName();
- value = so.getStringValue();
- break;
- case MATRIX:
- MatrixObject mo = (MatrixObject) dat;
- MatrixFormatMetaData md = (MatrixFormatMetaData) dat.getMetaData();
- MatrixCharacteristics mc = md.getMatrixCharacteristics();
- value = mo.getFileName();
- PDataPartitionFormat partFormat = (mo.getPartitionFormat()!=null) ? mo.getPartitionFormat() : PDataPartitionFormat.NONE;
- boolean inplace = mo.isUpdateInPlaceEnabled();
- matrixMetaData = new String[9];
- matrixMetaData[0] = String.valueOf( mc.getRows() );
- matrixMetaData[1] = String.valueOf( mc.getCols() );
- matrixMetaData[2] = String.valueOf( mc.getRowsPerBlock() );
- matrixMetaData[3] = String.valueOf( mc.getColsPerBlock() );
- matrixMetaData[4] = String.valueOf( mc.getNonZeros() );
- matrixMetaData[5] = InputInfo.inputInfoToString( md.getInputInfo() );
- matrixMetaData[6] = OutputInfo.outputInfoToString( md.getOutputInfo() );
- matrixMetaData[7] = String.valueOf( partFormat );
- matrixMetaData[8] = String.valueOf( inplace );
- break;
- default:
- throw new DMLRuntimeException("Unable to serialize datatype "+datatype);
- }
-
- //serialize data
- sb.append(name);
- sb.append(DATA_FIELD_DELIM);
- sb.append(datatype);
- sb.append(DATA_FIELD_DELIM);
- sb.append(valuetype);
- sb.append(DATA_FIELD_DELIM);
- sb.append(value);
- if( matrixMetaData != null )
- for( int i=0; i<matrixMetaData.length; i++ )
- {
- sb.append(DATA_FIELD_DELIM);
- sb.append(matrixMetaData[i]);
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- public static String serializeExecutionContext( ExecutionContext ec )
- throws DMLRuntimeException
- {
- String ret = null;
-
- if( ec != null )
- {
- LocalVariableMap vars = ec.getVariables();
- ret = serializeVariables( vars );
- }
- else
- {
- ret = EMPTY;
- }
-
- return ret;
- }
-
- /**
- *
- * @param inst
- * @return
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("all")
- public static String serializeInstructions( ArrayList<Instruction> inst )
- throws DMLRuntimeException
- {
- StringBuilder sb = new StringBuilder();
- int count = 0;
- for( Instruction linst : inst )
- {
- //check that only cp instruction are transmitted
- if( !( linst instanceof CPInstruction
- || linst instanceof SPInstruction
- || linst instanceof ExternalFunctionInvocationInstruction ) )
- {
- throw new DMLRuntimeException( NOT_SUPPORTED_MR_INSTRUCTION + " " +linst.getClass().getName()+"\n"+linst );
- }
-
- if( count > 0 )
- sb.append( ELEMENT_DELIM );
-
- sb.append( checkAndReplaceLiterals( linst.toString() ) );
- count++;
- }
-
- return sb.toString();
- }
-
- /**
- * Replacement of internal delimiters occurring in literals of instructions
- * in order to ensure robustness of serialization and parsing.
- * (e.g. print( "a,b" ) would break the parsing of instruction that internally
- * are separated with a "," )
- *
- * @param instStr
- * @return
- */
- public static String checkAndReplaceLiterals( String instStr )
- {
- String tmp = instStr;
-
- //1) check own delimiters (very unlikely due to special characters)
- if( tmp.contains(COMPONENTS_DELIM) ) {
- tmp = tmp.replaceAll(COMPONENTS_DELIM, ".");
- LOG.warn("Replaced special literal character sequence "+COMPONENTS_DELIM+" with '.'");
- }
-
- if( tmp.contains(ELEMENT_DELIM) ) {
- tmp = tmp.replaceAll(ELEMENT_DELIM, ".");
- LOG.warn("Replaced special literal character sequence "+ELEMENT_DELIM+" with '.'");
- }
-
- if( tmp.contains( LEVELIN ) ){
- tmp = tmp.replaceAll(LEVELIN, "("); // '\\' required if LEVELIN='{' because regex
- LOG.warn("Replaced special literal character sequence "+LEVELIN+" with '('");
- }
-
- if( tmp.contains(LEVELOUT) ){
- tmp = tmp.replaceAll(LEVELOUT, ")");
- LOG.warn("Replaced special literal character sequence "+LEVELOUT+" with ')'");
- }
-
- //NOTE: DATA_FIELD_DELIM and KEY_VALUE_DELIM not required
- //because those literals cannot occur in critical places.
-
- //2) check end tag of CDATA
- if( tmp.contains(PARFOR_CDATA_END) ){
- tmp = tmp.replaceAll(PARFOR_CDATA_END, "."); //prevent XML parsing issues in job.xml
- LOG.warn("Replaced special literal character sequence "+PARFOR_CDATA_END+" with '.'");
- }
-
- return tmp;
- }
-
-
- /**
- *
- * @param vars
- * @return
- */
- public static String serializeStringHashMap( HashMap<String,String> vars)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( Entry<String,String> e : vars.entrySet() )
- {
- if(count>0)
- sb.append( ELEMENT_DELIM );
- sb.append( e.getKey() );
- sb.append( KEY_VALUE_DELIM );
- sb.append( e.getValue() );
- count++;
- }
- return sb.toString();
- }
-
- /**
- *
- * @param vars
- * @return
- */
- public static String serializeStringCollection( Collection<String> set)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( String s : set )
- {
- if(count>0)
- sb.append( ", " );
- sb.append( s );
- count++;
- }
- return sb.toString();
- }
-
-
- /**
- *
- * @param vars
- * @return
- */
- public static String serializeStringArrayList( ArrayList<String> vars)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( String s : vars )
- {
- if(count>0)
- sb.append( ELEMENT_DELIM );
- sb.append( s );
- count++;
- }
- return sb.toString();
- }
-
- /**
- *
- * @param vars
- * @return
- */
- public static String serializeStringArray( String[] vars)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( String s : vars )
- {
- if(count>0)
- sb.append( ELEMENT_DELIM );
- if( s != null )
- sb.append( s );
- else
- sb.append( "null" );
-
- count++;
- }
- return sb.toString();
- }
-
- /**
- *
- * @param var
- * @return
- */
- public static String serializeDataIdentifiers( ArrayList<DataIdentifier> var)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( DataIdentifier dat : var )
- {
- if(count>0)
- sb.append( ELEMENT_DELIM );
- sb.append( serializeDataIdentifier(dat) );
- count++;
- }
- return sb.toString();
- }
-
- /**
- *
- * @param dat
- * @return
- */
- public static String serializeDataIdentifier( DataIdentifier dat )
- {
- // SCHEMA: <name>|<datatype>|<valuetype>
-
- StringBuilder sb = new StringBuilder();
- sb.append(dat.getName());
- sb.append(DATA_FIELD_DELIM);
- sb.append(dat.getDataType());
- sb.append(DATA_FIELD_DELIM);
- sb.append(dat.getValueType());
-
- return sb.toString();
- }
-
- /**
- *
- * @param pbs
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static String rSerializeFunctionProgramBlocks(HashMap<String,FunctionProgramBlock> pbs, HashSet<String> cand)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- StringBuilder sb = new StringBuilder();
-
- int count = 0;
- for( Entry<String,FunctionProgramBlock> pb : pbs.entrySet() )
- {
- if( !cand.contains(pb.getKey()) ) //skip function not included in the parfor body
- continue;
-
- if( count>0 )
- {
- sb.append( ELEMENT_DELIM );
- sb.append( NEWLINE );
- }
- sb.append( pb.getKey() );
- sb.append( KEY_VALUE_DELIM );
- sb.append( rSerializeProgramBlock( pb.getValue() ) );
-
- count++;
- }
- sb.append(NEWLINE);
- return sb.toString();
- }
-
- /**
- *
- * @param pbs
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static String rSerializeProgramBlocks(ArrayList<ProgramBlock> pbs)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- StringBuilder sb = new StringBuilder();
- int count = 0;
- for( ProgramBlock pb : pbs )
- {
- if( count>0 )
- {
- sb.append( ELEMENT_DELIM );
- sb.append(NEWLINE);
- }
- sb.append( rSerializeProgramBlock(pb) );
- count++;
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param pb
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static String rSerializeProgramBlock( ProgramBlock pb)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- StringBuilder sb = new StringBuilder();
-
- //handle header
- if( pb instanceof WhileProgramBlock )
- sb.append( PARFOR_PB_WHILE );
- else if ( pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock) )
- sb.append( PARFOR_PB_FOR );
- else if ( pb instanceof ParForProgramBlock )
- sb.append( PARFOR_PB_PARFOR );
- else if ( pb instanceof IfProgramBlock )
- sb.append( PARFOR_PB_IF );
- else if ( pb instanceof FunctionProgramBlock && !(pb instanceof ExternalFunctionProgramBlock) )
- sb.append( PARFOR_PB_FC );
- else if ( pb instanceof ExternalFunctionProgramBlock )
- sb.append( PARFOR_PB_EFC );
- else //all generic program blocks
- sb.append( PARFOR_PB_BEGIN );
-
- //handle variables (not required only on top level)
- /*sb.append( PARFOR_VARS_BEGIN );
- sb.append( serializeVariables( ) );
- sb.append( PARFOR_VARS_END );
- sb.append( COMPONENTS_DELIM );
- */
-
- //handle body
- if( pb instanceof WhileProgramBlock )
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( wpb.getPredicate() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( wpb.getPredicateResultVar() );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( wpb.getExitInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( wpb.getChildBlocks()) );
- sb.append( PARFOR_PBS_END );
- }
- else if ( pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock ) )
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- sb.append( serializeStringArray(fpb.getIterablePredicateVars()) );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( fpb.getFromInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( fpb.getToInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( fpb.getIncrementInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( fpb.getExitInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( fpb.getChildBlocks()) );
- sb.append( PARFOR_PBS_END );
- }
- else if ( pb instanceof ParForProgramBlock )
- {
- ParForProgramBlock pfpb = (ParForProgramBlock) pb;
-
- //check for nested remote ParFOR
- if( PExecMode.valueOf( pfpb.getParForParams().get( ParForStatementBlock.EXEC_MODE )) == PExecMode.REMOTE_MR )
- throw new DMLUnsupportedOperationException( NOT_SUPPORTED_MR_PARFOR );
-
- sb.append( serializeStringArray(pfpb.getIterablePredicateVars()) );
- sb.append( COMPONENTS_DELIM );
- sb.append( serializeStringArrayList( pfpb.getResultVariables()) );
- sb.append( COMPONENTS_DELIM );
- sb.append( serializeStringHashMap( pfpb.getParForParams()) ); //parameters of nested parfor
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( pfpb.getFromInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( pfpb.getToInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( pfpb.getIncrementInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( pfpb.getExitInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( pfpb.getChildBlocks() ) );
- sb.append( PARFOR_PBS_END );
- }
- else if ( pb instanceof IfProgramBlock )
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( ipb.getPredicate() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( ipb.getPredicateResultVar() );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( ipb.getExitInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( ipb.getChildBlocksIfBody() ) );
- sb.append( PARFOR_PBS_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( ipb.getChildBlocksElseBody() ) );
- sb.append( PARFOR_PBS_END );
- }
- else if( pb instanceof FunctionProgramBlock && !(pb instanceof ExternalFunctionProgramBlock) )
- {
- FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
-
- sb.append( serializeDataIdentifiers( fpb.getInputParams() ) );
- sb.append( COMPONENTS_DELIM );
- sb.append( serializeDataIdentifiers( fpb.getOutputParams() ) );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( fpb.getInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( fpb.getChildBlocks() ) );
- sb.append( PARFOR_PBS_END );
- sb.append( COMPONENTS_DELIM );
- }
- else if( pb instanceof ExternalFunctionProgramBlock )
- {
- if( !(pb instanceof ExternalFunctionProgramBlockCP) )
- {
- throw new DMLRuntimeException( NOT_SUPPORTED_EXTERNALFUNCTION_PB );
- }
-
- ExternalFunctionProgramBlockCP fpb = (ExternalFunctionProgramBlockCP) pb;
-
- sb.append( serializeDataIdentifiers( fpb.getInputParams() ) );
- sb.append( COMPONENTS_DELIM );
- sb.append( serializeDataIdentifiers( fpb.getOutputParams() ) );
- sb.append( COMPONENTS_DELIM );
- sb.append( serializeStringHashMap( fpb.getOtherParams() ) );
- sb.append( COMPONENTS_DELIM );
- sb.append( fpb.getBaseDir() );
- sb.append( COMPONENTS_DELIM );
-
- sb.append( PARFOR_INST_BEGIN );
- //create on construction anyway
- //sb.append( serializeInstructions( fpb.getInstructions() ) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( PARFOR_PBS_BEGIN );
- sb.append( rSerializeProgramBlocks( fpb.getChildBlocks() ) );
- sb.append( PARFOR_PBS_END );
- }
- else //all generic program blocks
- {
- sb.append( PARFOR_INST_BEGIN );
- sb.append( serializeInstructions( pb.getInstructions() ) );
- sb.append( PARFOR_INST_END );
- }
-
-
- //handle end
- sb.append( PARFOR_PB_END );
-
- return sb.toString();
- }
-
-
- ////////////////////////////////
- // PARSING
- ////////////////////////////////
-
-
- /**
- *
- * @param in
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ParForBody parseParForBody( String in, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ParForBody body = new ParForBody();
-
- //header elimination
- String tmpin = in.replaceAll(NEWLINE, ""); //normalization
- tmpin = tmpin.substring(PARFORBODY_BEGIN.length(),tmpin.length()-PARFORBODY_END.length()); //remove start/end
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(tmpin, COMPONENTS_DELIM);
-
- //handle DMLScript UUID (NOTE: set directly in DMLScript)
- //(master UUID is used for all nodes (in order to simply cleanup))
- DMLScript.setUUID( st.nextToken() );
-
- //handle DML config (NOTE: set directly in ConfigurationManager)
- String confStr = st.nextToken();
- JobConf job = ConfigurationManager.getCachedJobConf();
- if( !InfrastructureAnalyzer.isLocalMode(job) ) {
- DMLConfig config = DMLConfig.parseDMLConfig(confStr);
- ConfigurationManager.setConfig(config);
- ParForProgramBlock.initInternalConfigurations(config);
- }
-
- //handle additional configs
- String aconfs = st.nextToken();
- parseAndSetAdditionalConfigurations( aconfs );
-
- //handle program
- String progStr = st.nextToken();
- Program prog = parseProgram( progStr, id );
-
- //handle result variable names
- String rvarStr = st.nextToken();
- ArrayList<String> rvars = parseStringArrayList(rvarStr);
- body.setResultVarNames(rvars);
-
- //handle execution context
- String ecStr = st.nextToken();
- ExecutionContext ec = parseExecutionContext( ecStr, prog );
-
- //handle program blocks
- String spbs = st.nextToken();
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(spbs, prog, id);
-
- body.setChildBlocks( pbs );
- body.setEc( ec );
-
- return body;
- }
-
- /**
- *
- * @param in
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static Program parseProgram( String in, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PROG_BEGIN.length(),in.length()-PARFOR_PROG_END.length()).trim();
-
- Program prog = new Program();
- HashMap<String,FunctionProgramBlock> fc = parseFunctionProgramBlocks(lin, prog, id);
-
- for( Entry<String,FunctionProgramBlock> e : fc.entrySet() )
- {
- String[] keypart = e.getKey().split( Program.KEY_DELIM );
- String namespace = keypart[0];
- String name = keypart[1];
-
- prog.addFunctionProgramBlock(namespace, name, e.getValue());
- }
-
- return prog;
- }
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- */
- public static LocalVariableMap parseVariables(String in)
- throws DMLRuntimeException
- {
- LocalVariableMap ret = null;
-
- if( in.length()> PARFOR_VARS_BEGIN.length() + PARFOR_VARS_END.length())
- {
- String varStr = in.substring( PARFOR_VARS_BEGIN.length(),in.length()-PARFOR_VARS_END.length()).trim();
- ret = LocalVariableMap.deserialize(varStr);
- }
- else //empty input symbol table
- {
- ret = new LocalVariableMap();
- }
-
- return ret;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static HashMap<String,FunctionProgramBlock> parseFunctionProgramBlocks( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- HashMap<String,FunctionProgramBlock> ret = new HashMap<String, FunctionProgramBlock>();
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer( in, ELEMENT_DELIM );
-
- while( st.hasMoreTokens() )
- {
- String lvar = st.nextToken(); //with ID = CP_CHILD_THREAD+id for current use
-
- //put first copy into prog (for direct use)
- int index = lvar.indexOf( KEY_VALUE_DELIM );
- String tmp1 = lvar.substring(0, index); // + CP_CHILD_THREAD+id;
- String tmp2 = lvar.substring(index + 1);
- ret.put(tmp1, (FunctionProgramBlock)rParseProgramBlock(tmp2, prog, id));
-
- //put first copy into prog (for future deep copies)
- //index = lvar2.indexOf("=");
- //tmp1 = lvar2.substring(0, index);
- //tmp2 = lvar2.substring(index + 1);
- //ret.put(tmp1, (FunctionProgramBlock)rParseProgramBlock(tmp2, prog, id));
- }
-
- return ret;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ArrayList<ProgramBlock> rParseProgramBlocks(String in, Program prog, int id)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<ProgramBlock> pbs = new ArrayList<ProgramBlock>();
- String tmpdata = in.substring(PARFOR_PBS_BEGIN.length(),in.length()-PARFOR_PBS_END.length()); ;
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(tmpdata, ELEMENT_DELIM);
-
- while( st.hasMoreTokens() )
- {
- String tmp = st.nextToken();
- pbs.add( rParseProgramBlock( tmp, prog, id ) );
- }
-
- return pbs;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ProgramBlock rParseProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ProgramBlock pb = null;
-
- if( in.startsWith( PARFOR_PB_WHILE ) )
- pb = rParseWhileProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_FOR ) )
- pb = rParseForProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_PARFOR ) )
- pb = rParseParForProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_IF ) )
- pb = rParseIfProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_FC ) )
- pb = rParseFunctionProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_EFC ) )
- pb = rParseExternalFunctionProgramBlock( in, prog, id );
- else if ( in.startsWith(PARFOR_PB_BEGIN ) )
- pb = rParseGenericProgramBlock( in, prog, id );
- else
- throw new DMLUnsupportedOperationException( NOT_SUPPORTED_PB+" "+in );
-
- return pb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static WhileProgramBlock rParseWhileProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_WHILE.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //predicate instructions
- ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
- String var = st.nextToken();
-
- //exit instructions
- ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
-
- //program blocks
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
-
- WhileProgramBlock wpb = new WhileProgramBlock(prog,inst);
- wpb.setPredicateResultVar(var);
- wpb.setExitInstructions2(exit);
- wpb.setChildBlocks(pbs);
- //wpb.setVariables(vars);
-
- return wpb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ForProgramBlock rParseForProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_FOR.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //inputs
- String[] iterPredVars = parseStringArray(st.nextToken());
-
- //instructions
- ArrayList<Instruction> from = parseInstructions(st.nextToken(),id);
- ArrayList<Instruction> to = parseInstructions(st.nextToken(),id);
- ArrayList<Instruction> incr = parseInstructions(st.nextToken(),id);
-
- //exit instructions
- ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
-
- //program blocks
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
-
- ForProgramBlock fpb = new ForProgramBlock(prog, iterPredVars);
- fpb.setFromInstructions(from);
- fpb.setToInstructions(to);
- fpb.setIncrementInstructions(incr);
- fpb.setExitInstructions(exit);
- fpb.setChildBlocks(pbs);
- //fpb.setVariables(vars);
-
- return fpb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ParForProgramBlock rParseParForProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_PARFOR.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //inputs
- String[] iterPredVars = parseStringArray(st.nextToken());
- ArrayList<String> resultVars = parseStringArrayList(st.nextToken());
- HashMap<String,String> params = parseStringHashMap(st.nextToken());
-
- //instructions
- ArrayList<Instruction> from = parseInstructions(st.nextToken(), 0);
- ArrayList<Instruction> to = parseInstructions(st.nextToken(), 0);
- ArrayList<Instruction> incr = parseInstructions(st.nextToken(), 0);
-
- //exit instructions
- ArrayList<Instruction> exit = parseInstructions(st.nextToken(), 0);
-
- //program blocks //reset id to preinit state, replaced during exec
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, 0);
-
- ParForProgramBlock pfpb = new ParForProgramBlock(id, prog, iterPredVars, params);
- pfpb.disableOptimization(); //already done in top-level parfor
- pfpb.setResultVariables(resultVars);
- pfpb.setFromInstructions(from);
- pfpb.setToInstructions(to);
- pfpb.setIncrementInstructions(incr);
- pfpb.setExitInstructions(exit);
- pfpb.setChildBlocks(pbs);
- //pfpb.setVariables(vars);
-
- return pfpb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static IfProgramBlock rParseIfProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_IF.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //predicate instructions
- ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
- String var = st.nextToken();
-
- //exit instructions
- ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
-
- //program blocks: if and else
- ArrayList<ProgramBlock> pbs1 = rParseProgramBlocks(st.nextToken(), prog, id);
- ArrayList<ProgramBlock> pbs2 = rParseProgramBlocks(st.nextToken(), prog, id);
-
- IfProgramBlock ipb = new IfProgramBlock(prog,inst);
- ipb.setPredicateResultVar(var);
- ipb.setExitInstructions2(exit);
- ipb.setChildBlocksIfBody(pbs1);
- ipb.setChildBlocksElseBody(pbs2);
- //ipb.setVariables(vars);
-
- return ipb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static FunctionProgramBlock rParseFunctionProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_FC.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //inputs and outputs
- ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
- ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
-
- //instructions
- ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
-
- //program blocks
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
-
- ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(dat1);
- ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(dat2);
- FunctionProgramBlock fpb = new FunctionProgramBlock(prog, tmp1, tmp2);
- fpb.setInstructions(inst);
- fpb.setChildBlocks(pbs);
- //fpb.setVariables(vars);
-
- return fpb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ExternalFunctionProgramBlock rParseExternalFunctionProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_EFC.length(),in.length()-PARFOR_PB_END.length());
- HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
-
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
- //inputs, outputs and params
- ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
- ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
- HashMap<String,String> dat3 = parseStringHashMap(st.nextToken());
-
- //basedir
- String basedir = st.nextToken();
-
- //instructions (required for removing INST BEGIN, END)
- parseInstructions(st.nextToken(),id);
-
- //program blocks
- ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
-
- ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(dat1);
- ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(dat2);
-
- //only CP external functions, because no nested MR jobs for reblocks
- ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir);
- //efpb.setInstructions(inst);
- efpb.setChildBlocks(pbs);
- //efpb.setVariables(vars);
-
- return efpb;
- }
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ProgramBlock rParseGenericProgramBlock( String in, Program prog, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String lin = in.substring( PARFOR_PB_BEGIN.length(),in.length()-PARFOR_PB_END.length());
- StringTokenizer st = new StringTokenizer(lin,COMPONENTS_DELIM);
-
- ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
-
- ProgramBlock pb = new ProgramBlock(prog);
- pb.setInstructions(inst);
-
- return pb;
- }
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ArrayList<Instruction> parseInstructions( String in, int id )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> insts = new ArrayList<Instruction>();
-
- String lin = in.substring( PARFOR_INST_BEGIN.length(),in.length()-PARFOR_INST_END.length());
- StringTokenizer st = new StringTokenizer(lin, ELEMENT_DELIM);
- while(st.hasMoreTokens())
- {
- //Note that at this point only CP instructions and External function instruction can occur
- String instStr = st.nextToken();
-
- try
- {
- Instruction tmpinst = CPInstructionParser.parseSingleInstruction(instStr);
- tmpinst = saveReplaceThreadID(tmpinst, CP_ROOT_THREAD_ID, CP_CHILD_THREAD+id );
- insts.add( tmpinst );
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Failed to parse instruction: " + instStr, ex);
- }
- }
-
- return insts;
- }
-
- /**
- *
- * @param in
- * @return
- */
- public static HashMap<String,String> parseStringHashMap( String in )
- {
- HashMap<String,String> vars = new HashMap<String, String>();
- StringTokenizer st = new StringTokenizer(in,ELEMENT_DELIM);
- while( st.hasMoreTokens() )
- {
- String lin = st.nextToken();
- int index = lin.indexOf( KEY_VALUE_DELIM );
- String tmp1 = lin.substring(0, index);
- String tmp2 = lin.substring(index + 1);
- vars.put(tmp1, tmp2);
- }
-
- return vars;
- }
-
- /**
- *
- * @param in
- * @return
- */
- public static ArrayList<String> parseStringArrayList( String in )
- {
- ArrayList<String> vars = new ArrayList<String>();
- StringTokenizer st = new StringTokenizer(in,ELEMENT_DELIM);
- while( st.hasMoreTokens() )
- {
- String tmp = st.nextToken();
- vars.add(tmp);
- }
-
- return vars;
- }
-
- /**
- *
- * @param in
- * @return
- */
- public static String[] parseStringArray( String in )
- {
- StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM);
- int len = st.countTokens();
- String[] a = new String[len];
- for( int i=0; i<len; i++ )
- {
- String tmp = st.nextToken();
- if( tmp.equals("null") )
- a[i] = null;
- else
- a[i] = tmp;
- }
- return a;
- }
-
- /**
- *
- * @param in
- * @return
- */
- public static ArrayList<DataIdentifier> parseDataIdentifiers( String in )
- {
- ArrayList<DataIdentifier> vars = new ArrayList<DataIdentifier>();
- StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM);
- while( st.hasMoreTokens() )
- {
- String tmp = st.nextToken();
- DataIdentifier dat = parseDataIdentifier( tmp );
- vars.add(dat);
- }
-
- return vars;
- }
-
- /**
- *
- * @param in
- * @return
- */
- public static DataIdentifier parseDataIdentifier( String in )
- {
- StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
- String name = st.nextToken();
- DataType dt = DataType.valueOf(st.nextToken());
- ValueType vt = ValueType.valueOf(st.nextToken());
-
- DataIdentifier dat = new DataIdentifier(name);
- dat.setDataType(dt);
- dat.setValueType(vt);
-
- return dat;
- }
-
- /**
- * NOTE: MRJobConfiguration cannot be used for the general case because program blocks and
- * related symbol tables can be hierarchically structured.
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- */
- public static Object[] parseDataObject(String in)
- throws DMLRuntimeException
- {
- Object[] ret = new Object[2];
-
- StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM );
- String name = st.nextToken();
- DataType datatype = DataType.valueOf( st.nextToken() );
- ValueType valuetype = ValueType.valueOf( st.nextToken() );
- String valString = st.hasMoreTokens() ? st.nextToken() : "";
- Data dat = null;
- switch( datatype )
- {
- case SCALAR:
- {
- switch ( valuetype )
- {
- case INT:
- long value1 = Long.parseLong(valString);
- dat = new IntObject(name,value1);
- break;
- case DOUBLE:
- double value2 = Double.parseDouble(valString);
- dat = new DoubleObject(name,value2);
- break;
- case BOOLEAN:
- boolean value3 = Boolean.parseBoolean(valString);
- dat = new BooleanObject(name,value3);
- break;
- case STRING:
- dat = new StringObject(name,valString);
- break;
- default:
- throw new DMLRuntimeException("Unable to parse valuetype "+valuetype);
- }
- break;
- }
- case MATRIX:
- {
- MatrixObject mo = new MatrixObject(valuetype,valString);
- long rows = Long.parseLong( st.nextToken() );
- long cols = Long.parseLong( st.nextToken() );
- int brows = Integer.parseInt( st.nextToken() );
- int bcols = Integer.parseInt( st.nextToken() );
- long nnz = Long.parseLong( st.nextToken() );
- InputInfo iin = InputInfo.stringToInputInfo( st.nextToken() );
- OutputInfo oin = OutputInfo.stringToOutputInfo( st.nextToken() );
- PDataPartitionFormat partFormat = PDataPartitionFormat.valueOf( st.nextToken() );
- boolean inplace = Boolean.parseBoolean( st.nextToken() );
- MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
- MatrixFormatMetaData md = new MatrixFormatMetaData( mc, oin, iin );
- mo.setMetaData( md );
- mo.setVarName( name );
- if( partFormat!=PDataPartitionFormat.NONE )
- mo.setPartitioned( partFormat, -1 ); //TODO once we support BLOCKWISE_N we should support it here as well
- mo.enableUpdateInPlace(inplace);
- dat = mo;
- break;
- }
- default:
- throw new DMLRuntimeException("Unable to parse datatype "+datatype);
- }
-
- ret[0] = name;
- ret[1] = dat;
- return ret;
- }
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static ExecutionContext parseExecutionContext(String in, Program prog)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ExecutionContext ec = null;
-
- String lin = in.substring(PARFOR_EC_BEGIN.length(),in.length()-PARFOR_EC_END.length()).trim();
-
- if( !lin.equals( EMPTY ) )
- {
- LocalVariableMap vars = parseVariables(lin);
- ec = ExecutionContextFactory.createContext( false, prog );
- ec.setVariables(vars);
- }
-
- return ec;
- }
-
- public static void parseAndSetAdditionalConfigurations(String conf)
- {
- //set statistics flag
- String[] statsFlag = conf.split("=");
- DMLScript.STATISTICS = Boolean.parseBoolean(statsFlag[1]);
- }
-
- //////////
- // CUSTOM SAFE LITERAL REPLACEMENT
-
- /**
- * In-place replacement of thread ids in filenames, functions names etc
- *
- * @param instInst
- * @param pattern
- * @param replacement
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static Instruction saveReplaceThreadID( Instruction inst, String pattern, String replacement )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //currently known, relevant instructions: createvar, rand, seq, extfunct,
- if( inst instanceof MRJobInstruction )
- {
- //update dims file, and internal string representations of rand/seq instructions
- MRJobInstruction mrinst = (MRJobInstruction)inst;
- mrinst.updateInstructionThreadID(pattern, replacement);
- }
- else if ( inst instanceof VariableCPInstruction ) //createvar, setfilename
- {
- //update in-memory representation
- inst.updateInstructionThreadID(pattern, replacement);
- }
- //NOTE> //Rand, seq in CP not required
- //else if( inst.toString().contains(pattern) )
- // throw new DMLRuntimeException( "DEBUG: Missed thread id replacement: "+inst );
-
- return inst;
- }
-
- public static String saveReplaceFilenameThreadID(String fname, String pattern, String replace)
- {
- //save replace necessary in order to account for the possibility that read variables have our prefix in the absolute path
- //replace the last match only, because (1) we have at most one _t0 and (2) always concatenated to the end.
- int pos = fname.lastIndexOf(pattern);
- if( pos < 0 )
- return fname;
- return fname.substring(0, pos) + replace + fname.substring(pos+pattern.length());
- }
-
-
- //////////
- // CUSTOM HIERARCHICAL TOKENIZER
-
-
- /**
- * Custom StringTokenizer for splitting strings of hierarchies. The basic idea is to
- * search for delim-Strings on the same hierarchy level, while delims of lower hierarchy
- * levels are skipped.
- *
- */
- private static class HierarchyAwareStringTokenizer //extends StringTokenizer
- {
- private String _str = null;
- private String _del = null;
- private int _off = -1;
-
- public HierarchyAwareStringTokenizer( String in, String delim )
- {
- //super(in);
- _str = in;
- _del = delim;
- _off = delim.length();
- }
-
- public boolean hasMoreTokens()
- {
- return (_str.length() > 0);
- }
-
- public String nextToken()
- {
- int nextDelim = determineNextSameLevelIndexOf(_str, _del);
- String token = null;
- if(nextDelim < 0)
- {
- nextDelim = _str.length();
- _off = 0;
- }
- token = _str.substring(0,nextDelim);
- _str = _str.substring( nextDelim + _off );
- return token;
- }
-
- private int determineNextSameLevelIndexOf( String data, String pattern )
- {
- String tmpdata = data;
- int index = 0;
- int count = 0;
- int off=0,i1,i2,i3,min;
-
- while(true)
- {
- i1 = tmpdata.indexOf(pattern);
- i2 = tmpdata.indexOf(LEVELIN);
- i3 = tmpdata.indexOf(LEVELOUT);
-
- if( i1 < 0 ) return i1; //no pattern found at all
-
- min = i1; //min >= 0 by definition
- if( i2 >= 0 ) min = Math.min(min, i2);
- if( i3 >= 0 ) min = Math.min(min, i3);
-
- //stack maintenance
- if( i1 == min && count == 0 )
- return index+i1;
- else if( i2 == min )
- {
- count++;
- off = LEVELIN.length();
- }
- else if( i3 == min )
- {
- count--;
- off = LEVELOUT.length();
- }
-
- //prune investigated string
- index += min+off;
- tmpdata = tmpdata.substring(min+off);
- }
- }
- }
-}
[43/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java b/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
deleted file mode 100644
index 90d5ec0..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-
-/**
- *
- *
- */
-public class DataGenOp extends Hop implements MultiThreadedHop
-{
-
- public static final long UNSPECIFIED_SEED = -1;
-
- // defines the specific data generation method
- private DataGenMethod _op;
- private int _maxNumThreads = -1; //-1 for unlimited
-
- /**
- * List of "named" input parameters. They are maintained as a hashmap:
- * parameter names (String) are mapped as indices (Integer) into getInput()
- * arraylist.
- *
- * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
- * that is associated with parameterName.
- */
- private HashMap<String, Integer> _paramIndexMap = new HashMap<String, Integer>();
-
-
- /** target identifier which will hold the random object */
- private DataIdentifier _id;
-
- //Rand-specific attributes
-
- /** sparsity of the random object, this is used for mem estimate */
- private double _sparsity = -1;
- /** base directory for temp file (e.g., input seeds)*/
- private String _baseDir;
-
- //seq-specific attributes (used for recompile/recompile)
- private double _incr = Double.MAX_VALUE;
-
-
- private DataGenOp() {
- //default constructor for clone
- }
-
- /**
- * <p>Creates a new Rand HOP.</p>
- *
- * @param id the target identifier
- * @param inputParameters HashMap of the input parameters for Rand Hop
- */
- public DataGenOp(DataGenMethod mthd, DataIdentifier id, HashMap<String, Hop> inputParameters)
- {
- super(id.getName(), DataType.MATRIX, ValueType.DOUBLE);
-
- _id = id;
- _op = mthd;
-
- int index = 0;
- for( Entry<String, Hop> e: inputParameters.entrySet() ) {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
- if ( mthd == DataGenMethod.RAND )
- _sparsity = Double.valueOf(((LiteralOp)inputParameters.get(DataExpression.RAND_SPARSITY)).getName());
-
- //generate base dir
- String scratch = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
- _baseDir = scratch + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- Lop.FILE_SEPARATOR + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR;
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- @Override
- public String getOpString() {
- return "dg(" + _op.toString().toLowerCase() +")";
- }
-
- public DataGenMethod getOp() {
- return _op;
- }
-
- @Override
- public void setMaxNumThreads( int k ) {
- _maxNumThreads = k;
- }
-
- @Override
- public int getMaxNumThreads() {
- return _maxNumThreads;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
-
- HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
- for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
- if( cur.getKey().equals(DataExpression.RAND_ROWS) && _dim1>0 )
- inputLops.put(cur.getKey(), new LiteralOp(_dim1).constructLops());
- else if( cur.getKey().equals(DataExpression.RAND_COLS) && _dim2>0 )
- inputLops.put(cur.getKey(), new LiteralOp(_dim2).constructLops());
- else
- inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
- }
-
- DataGen rnd = new DataGen(_op, _id, inputLops,_baseDir,
- getDataType(), getValueType(), et);
-
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- rnd.setNumThreads(k);
-
- rnd.getOutputParameters().setDimensions(
- getDim1(), getDim2(),
- //robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
- (getRowsInBlock()>0)?getRowsInBlock():DMLTranslator.DMLBlockSize,
- (getColsInBlock()>0)?getColsInBlock():DMLTranslator.DMLBlockSize,
- //actual rand nnz might differ (in cp/mr they are corrected after execution)
- (_op==DataGenMethod.RAND && et==ExecType.SPARK && getNnz()!=0) ? -1 : getNnz() );
-
- setLineNumbers(rnd);
- setLops(rnd);
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- public void printMe() throws HopsException
- {
- if (LOG.isDebugEnabled()){
- if(getVisited() != VisitStatus.DONE)
- {
- super.printMe();
- }
-
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if ( _op == DataGenMethod.RAND ) {
- if( hasConstantValue(0.0) ) { //if empty block
- ret = OptimizerUtils.estimateSizeEmptyBlock(dim1, dim2);
- }
- else {
- //sparsity-aware estimation (dependent on sparse generation approach); for pure dense generation
- //we would need to disable sparsity-awareness and estimate via sparsity=1.0
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
- }
- }
- else {
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
- }
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( _op == DataGenMethod.RAND && dimsKnown() ) {
- long numBlocks = (long) (Math.ceil((double)dim1/DMLTranslator.DMLBlockSize) * Math.ceil((double)dim2/DMLTranslator.DMLBlockSize));
- return 32 + numBlocks*8.0; // 32 bytes of overhead for an array of long & numBlocks long values.
- }
- else
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- //infer rows and
- if( (_op == DataGenMethod.RAND || _op == DataGenMethod.SINIT ) &&
- OptimizerUtils.ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION )
- {
- long dim1 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_ROWS)), memo);
- long dim2 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_COLS)), memo);
- long nnz = (long)(_sparsity * dim1 * dim2);
- if( dim1>0 && dim2>0 )
- return new long[]{ dim1, dim2, nnz };
- }
- else if ( _op == DataGenMethod.SEQ )
- {
- Hop from = getInput().get(_paramIndexMap.get(Statement.SEQ_FROM));
- Hop to = getInput().get(_paramIndexMap.get(Statement.SEQ_TO));
- Hop incr = getInput().get(_paramIndexMap.get(Statement.SEQ_INCR));
- //in order to ensure correctness we also need to know from and incr
- //here, we check for the common case of seq(1,x), i.e. from=1, incr=1
- if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1
- && incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==1 )
- {
- long toVal = computeDimParameterInformation(to, memo);
- if( toVal > 0 )
- return new long[]{ toVal, 1, -1 };
- }
- //here, we check for the common case of seq(1,x,?), i.e., from=1, to=x, b(seqincr) operator
- if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1
- && incr instanceof BinaryOp && ((BinaryOp)incr).getOp() == Hop.OpOp2.SEQINCR ) //implicit 1
- {
- long toVal = computeDimParameterInformation(to, memo);
- if( toVal > 0 )
- return new long[]{ toVal, 1, -1 };
- }
- //here, we check for the common case of seq(x,1,-1), i.e. from=x, to=1 incr=-1
- if( to instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)to)==1
- && incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==-1 )
- {
- long fromVal = computeDimParameterInformation(from, memo);
- if( fromVal > 0 )
- return new long[]{ fromVal, 1, -1 };
- }
- }
-
- return null;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- _etype = _etypeForced;
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if (this.areDimsBelowThreshold() || this.isVector())
- _etype = ExecType.CP;
- else
- _etype = REMOTE;
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- //always force string initialization into CP (not supported in MR)
- //similarly, sample is currently not supported in MR either
- if( _op == DataGenMethod.SINIT )
- _etype = ExecType.CP;
-
- //workaround until sample supported in MR
- if( _op == DataGenMethod.SAMPLE && _etype == ExecType.MR )
- _etype = ExecType.CP;
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = null;
- Hop input2 = null;
- Hop input3 = null;
-
- if ( _op == DataGenMethod.RAND || _op == DataGenMethod.SINIT )
- {
- input1 = getInput().get(_paramIndexMap.get(DataExpression.RAND_ROWS)); //rows
- input2 = getInput().get(_paramIndexMap.get(DataExpression.RAND_COLS)); //cols
-
- //refresh rows information
- refreshRowsParameterInformation(input1);
-
- //refresh cols information
- refreshColsParameterInformation(input2);
- }
- else if (_op == DataGenMethod.SEQ )
- {
- input1 = getInput().get(_paramIndexMap.get(Statement.SEQ_FROM));
- input2 = getInput().get(_paramIndexMap.get(Statement.SEQ_TO));
- input3 = getInput().get(_paramIndexMap.get(Statement.SEQ_INCR));
-
- double from = computeBoundsInformation(input1);
- boolean fromKnown = (from != Double.MAX_VALUE);
-
- double to = computeBoundsInformation(input2);
- boolean toKnown = (to != Double.MAX_VALUE);
-
- double incr = computeBoundsInformation(input3);
- boolean incrKnown = (incr != Double.MAX_VALUE);
-
- if( !incrKnown && input3 instanceof BinaryOp //special case for incr
- && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR && fromKnown && toKnown)
- {
- incr = ( from >= to ) ? -1 : 1;
- incrKnown = true;
- }
-
- if ( fromKnown && toKnown && incrKnown ) {
- setDim1(1 + (long)Math.floor(((double)(to-from))/incr));
- setDim2(1);
- _incr = incr;
- }
- }
-
- //refresh nnz information (for seq, sparsity is always -1)
- if( _op == DataGenMethod.RAND && hasConstantValue(0.0) )
- _nnz = 0;
- else if ( dimsKnown() && _sparsity>=0 ) //general case
- _nnz = (long) (_sparsity * _dim1 * _dim2);
- }
-
-
- public HashMap<String, Integer> getParamIndexMap()
- {
- return _paramIndexMap;
- }
-
- public int getParamIndex(String key)
- {
- return _paramIndexMap.get(key);
- }
-
- /**
- *
- * @return
- */
- public boolean hasConstantValue()
- {
- //string initialization does not exhibit constant values
- if( _op == DataGenMethod.SINIT )
- return false;
-
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN)); //min
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX)); //max
- Hop sparsity = getInput().get(_paramIndexMap.get(DataExpression.RAND_SPARSITY)); //sparsity
-
- //literal value comparison
- if( min instanceof LiteralOp && max instanceof LiteralOp && sparsity instanceof LiteralOp){
- try{
- double minVal = HopRewriteUtils.getDoubleValue((LiteralOp)min);
- double maxVal = HopRewriteUtils.getDoubleValue((LiteralOp)max);
- double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
- return (sp==1.0 && minVal == maxVal);
- }
- catch(Exception ex)
- {
- return false;
- }
- }
- //reference comparison (based on common subexpression elimination)
- else if ( min == max && sparsity instanceof LiteralOp ) {
- try {
- double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
- return (sp==1.0);
- }
- catch(Exception ex)
- {
- return false;
- }
- }
-
- return false;
- }
-
- /**
- *
- * @param val
- * @return
- */
- public boolean hasConstantValue(double val)
- {
- //string initialization does not exhibit constant values
- if( _op == DataGenMethod.SINIT )
- return false;
-
- boolean ret = false;
-
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN)); //min
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX)); //max
-
- //literal value comparison
- if( min instanceof LiteralOp && max instanceof LiteralOp){
- double minVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)min);
- double maxVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)max);
- ret = (minVal == val && maxVal == val);
- }
-
- //sparsity awareness if requires
- if( ret && val != 0 ) {
- Hop sparsity = getInput().get(_paramIndexMap.get(DataExpression.RAND_SPARSITY)); //sparsity
- ret &= (sparsity == null || sparsity instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)sparsity) == 1);
- }
-
- return ret;
- }
-
- public void setIncrementValue(double incr)
- {
- _incr = incr;
- }
-
- public double getIncrementValue()
- {
- return _incr;
- }
-
- public static long generateRandomSeed()
- {
- return System.nanoTime();
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object clone() throws CloneNotSupportedException
- {
- DataGenOp ret = new DataGenOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._id = _id;
- ret._sparsity = _sparsity;
- ret._baseDir = _baseDir;
- ret._paramIndexMap = (HashMap<String, Integer>) _paramIndexMap.clone();
- ret._maxNumThreads = _maxNumThreads;
- //note: no deep cp of params since read-only
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof DataGenOp) )
- return false;
-
- DataGenOp that2 = (DataGenOp)that;
- boolean ret = ( _op == that2._op
- && _sparsity == that2._sparsity
- && _baseDir.equals(that2._baseDir)
- && _paramIndexMap!=null && that2._paramIndexMap!=null
- && _maxNumThreads == that2._maxNumThreads );
-
- if( ret )
- {
- for( Entry<String,Integer> e : _paramIndexMap.entrySet() )
- {
- String key1 = e.getKey();
- int pos1 = e.getValue();
- int pos2 = that2._paramIndexMap.get(key1);
- ret &= ( that2.getInput().get(pos2)!=null
- && getInput().get(pos1) == that2.getInput().get(pos2) );
- }
-
- //special case for rand seed (no CSE if unspecified seed because runtime generated)
- //note: if min and max is constant, we can safely merge those hops
- if( _op == DataGenMethod.RAND || _op == DataGenMethod.SINIT ){
- Hop seed = getInput().get(_paramIndexMap.get(DataExpression.RAND_SEED));
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN));
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX));
- if( seed.getName().equals(String.valueOf(DataGenOp.UNSPECIFIED_SEED)) && min != max )
- ret = false;
- }
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/DataOp.java b/src/main/java/com/ibm/bi/dml/hops/DataOp.java
deleted file mode 100644
index 3975ed9..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/DataOp.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-
-public class DataOp extends Hop
-{
-
- private DataOpTypes _dataop;
- private String _fileName = null;
-
- //read dataop properties
- private FileFormatTypes _inFormat = FileFormatTypes.TEXT;
- private long _inRowsInBlock = -1;
- private long _inColsInBlock = -1;
-
- private boolean _recompileRead = true;
-
- /**
- * List of "named" input parameters. They are maintained as a hashmap:
- * parameter names (String) are mapped as indices (Integer) into getInput()
- * arraylist.
- *
- * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
- * that is associated with parameterName.
- */
- private HashMap<String, Integer> _paramIndexMap = new HashMap<String, Integer>();
-
- private DataOp() {
- //default constructor for clone
- }
-
- /**
- * READ operation for Matrix w/ dim1, dim2.
- * This constructor does not support any expression in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt, DataOpTypes dop,
- String fname, long dim1, long dim2, long nnz, long rowsPerBlock, long colsPerBlock) {
- super(l, dt, vt);
- _dataop = dop;
-
- _fileName = fname;
- setDim1(dim1);
- setDim2(dim2);
- setRowsInBlock(rowsPerBlock);
- setColsInBlock(colsPerBlock);
- setNnz(nnz);
-
- if( dop == DataOpTypes.TRANSIENTREAD )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- /**
- * READ operation for Matrix
- * This constructor supports expressions in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt,
- DataOpTypes dop, HashMap<String, Hop> inputParameters) {
- super(l, dt, vt);
-
- _dataop = dop;
-
- int index = 0;
- for( Entry<String, Hop> e : inputParameters.entrySet() )
- {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
- if (dop == DataOpTypes.TRANSIENTREAD ){
- setInputFormatType(FileFormatTypes.BINARY);
- }
- }
-
- // WRITE operation
- // This constructor does not support any expression in parameters
- public DataOp(String l, DataType dt, ValueType vt, Hop in,
- DataOpTypes dop, String fname) {
- super(l, dt, vt);
- _dataop = dop;
- getInput().add(0, in);
- in.getParent().add(this);
- _fileName = fname;
-
- if (dop == DataOpTypes.TRANSIENTWRITE || dop == DataOpTypes.FUNCTIONOUTPUT )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- // CHECKPOINT operation
- // This constructor does not support any expression in parameters
- public DataOp(String l, DataType dt, ValueType vt, Hop in,
- LiteralOp level, DataOpTypes dop, String fname) {
- super(l, dt, vt);
- _dataop = dop;
- getInput().add(0, in);
- getInput().add(1, level);
- in.getParent().add(this);
- level.getParent().add(this);
- _fileName = fname;
-
- if (dop == DataOpTypes.TRANSIENTWRITE || dop == DataOpTypes.FUNCTIONOUTPUT )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
-
- /**
- * WRITE operation for Matrix
- * This constructor supports expression in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt,
- DataOpTypes dop, Hop in, HashMap<String, Hop> inputParameters) {
- super(l, dt, vt);
-
- _dataop = dop;
-
- getInput().add(0, in);
- in.getParent().add(this);
-
- if (inputParameters != null){
- int index = 1;
- for( Entry<String, Hop> e : inputParameters.entrySet() )
- {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
-
- }
-
- if (dop == DataOpTypes.TRANSIENTWRITE)
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- public DataOpTypes getDataOpType()
- {
- return _dataop;
- }
-
- public void setDataOpType( DataOpTypes type )
- {
- _dataop = type;
- }
-
- public void setOutputParams(long dim1, long dim2, long nnz, long rowsPerBlock, long colsPerBlock) {
- setDim1(dim1);
- setDim2(dim2);
- setNnz(nnz);
- setRowsInBlock(rowsPerBlock);
- setColsInBlock(colsPerBlock);
- }
-
- public void setFileName(String fn) {
- _fileName = fn;
- }
-
- public String getFileName() {
- return _fileName;
- }
-
- public int getParameterIndex(String name)
- {
- return _paramIndexMap.get(name);
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
- Lop l = null;
-
- // construct lops for all input parameters
- HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
- for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
- inputLops.put(cur.getKey(), getInput().get(cur.getValue())
- .constructLops());
- }
-
- // Create the lop
- switch(_dataop)
- {
- case TRANSIENTREAD:
- l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- setOutputDimensions(l);
- break;
-
- case PERSISTENTREAD:
- l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null,
- getDataType(), getValueType(), false, getInputFormatType());
- l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inRowsInBlock, _inColsInBlock, getNnz());
- break;
-
- case PERSISTENTWRITE:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), false, getInputFormatType());
- ((Data)l).setExecType(et);
- setOutputDimensions(l);
- break;
-
- case TRANSIENTWRITE:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- setOutputDimensions(l);
- break;
-
- case FUNCTIONOUTPUT:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- ((Data)l).setExecType(et);
- setOutputDimensions(l);
- break;
-
- default:
- throw new LopsException("Invalid operation type for Data LOP: " + _dataop);
- }
-
- setLineNumbers(l);
- setLops(l);
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
-
- }
-
- public void setInputFormatType(FileFormatTypes ft) {
- _inFormat = ft;
- }
-
- public FileFormatTypes getInputFormatType() {
- return _inFormat;
- }
-
- public void setInputBlockSizes( long brlen, long bclen ){
- setInputRowsInBlock(brlen);
- setInputColsInBlock(bclen);
- }
-
- public void setInputRowsInBlock( long brlen ){
- _inRowsInBlock = brlen;
- }
-
- public long getInputRowsInBlock(){
- return _inRowsInBlock;
- }
-
- public void setInputColsInBlock( long bclen ){
- _inColsInBlock = bclen;
- }
-
- public long getInputColsInBlock(){
- return _inColsInBlock;
- }
-
- public boolean isRead()
- {
- return( _dataop == DataOpTypes.PERSISTENTREAD || _dataop == DataOpTypes.TRANSIENTREAD );
- }
-
- public boolean isWrite()
- {
- return( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE );
- }
-
- public boolean isPersistentReadWrite()
- {
- return( _dataop == DataOpTypes.PERSISTENTREAD || _dataop == DataOpTypes.PERSISTENTWRITE );
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += HopsData2String.get(_dataop);
- s += " "+getName();
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" DataOp: " + _dataop);
- if (_fileName != null) {
- LOG.debug(" file: " + _fileName);
- }
- LOG.debug(" format: " + getInputFormatType());
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return false;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if ( getDataType() == DataType.SCALAR )
- {
- switch( getValueType() )
- {
- case INT:
- ret = OptimizerUtils.INT_SIZE; break;
- case DOUBLE:
- ret = OptimizerUtils.DOUBLE_SIZE; break;
- case BOOLEAN:
- ret = OptimizerUtils.BOOLEAN_SIZE; break;
- case STRING:
- // by default, it estimates the size of string[100]
- ret = 100 * OptimizerUtils.CHAR_SIZE; break;
- case OBJECT:
- ret = OptimizerUtils.DEFAULT_SIZE; break;
- default:
- ret = 0;
- }
- }
- else //MATRIX
- {
- if( _dataop == DataOpTypes.PERSISTENTREAD
- || _dataop == DataOpTypes.TRANSIENTREAD )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
- // output memory estimate is not required for "write" nodes (just input)
- }
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- return LocalFileUtils.BUFFER_SIZE;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- if( _dataop == DataOpTypes.PERSISTENTWRITE
- || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- MatrixCharacteristics mc = memo.getAllInputStats(getInput().get(0));
- if( mc.dimsKnown() )
- ret = new long[]{ mc.getRows(), mc.getCols(), mc.getNonZeros() };
- }
- else if( _dataop == DataOpTypes.TRANSIENTREAD )
- {
- //prepare statistics, passed from cross-dag transient writes
- MatrixCharacteristics mc = memo.getAllInputStats(this);
- if( mc.dimsKnown() )
- ret = new long[]{ mc.getRows(), mc.getCols(), mc.getNonZeros() };
- }
-
- return ret;
- }
-
-
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- //MB: find exec type has two meanings here: (1) for write it means the actual
- //exec type, while (2) for read it affects the recompilation decision as needed
- //for example for sum(X) where the memory consumption is solely determined by the DataOp
-
- ExecType letype = (OptimizerUtils.isMemoryBasedOptLevel()) ? findExecTypeByMemEstimate() : null;
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //NOTE: independent of etype executed in MR (piggybacked) if input to persistent write is MR
- if( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- checkAndSetForcedPlatform();
-
- //additional check for write only
- if( getDataType()==DataType.SCALAR )
- _etypeForced = ExecType.CP;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = letype;
- }
- else if ( getInput().get(0).areDimsBelowThreshold() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
- }
- else //READ
- {
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && letype==REMOTE
- && (_recompileRead || _requiresCheckpoint) )
- {
- setRequiresRecompile();
- }
-
- _etype = letype;
- }
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- if( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- Hop input1 = getInput().get(0);
- setDim1(input1.getDim1());
- setDim2(input1.getDim2());
- setNnz(input1.getNnz());
- }
- else //READ
- {
- //do nothing; dimensions updated via set output params
- }
- }
-
-
- /**
- * Explicitly disables recompilation of transient reads, this additional information
- * is required because requiresRecompile is set in a top-down manner, hence any value
- * set from a consuming operating would be overwritten by opFindExecType.
- */
- public void disableRecompileRead()
- {
- _recompileRead = false;
- }
-
-
- @Override
- @SuppressWarnings("unchecked")
- public Object clone() throws CloneNotSupportedException
- {
- DataOp ret = new DataOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._dataop = _dataop;
- ret._fileName = _fileName;
- ret._inFormat = _inFormat;
- ret._inRowsInBlock = _inRowsInBlock;
- ret._inColsInBlock = _inColsInBlock;
- ret._recompileRead = _recompileRead;
- ret._paramIndexMap = (HashMap<String, Integer>) _paramIndexMap.clone();
- //note: no deep cp of params since read-only
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java b/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
deleted file mode 100644
index 3fe3c61..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.FunctionCallCP;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
-
-/**
- * This FunctionOp represents the call to a DML-bodied or external function.
- *
- * Note: Currently, we support expressions in function arguments but no function calls
- * in expressions.
- */
-public class FunctionOp extends Hop
-{
-
- public static String OPSTRING = "extfunct";
-
- public enum FunctionType{
- DML,
- EXTERNAL_MEM,
- EXTERNAL_FILE,
- MULTIRETURN_BUILTIN,
- UNKNOWN
- }
-
- private FunctionType _type = null;
- private String _fnamespace = null;
- private String _fname = null;
- private String[] _outputs = null;
- private ArrayList<Hop> _outputHops = null;
-
- private FunctionOp() {
- //default constructor for clone
- }
-
- public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, ArrayList<Hop> outputHops) {
- this(type, fnamespace, fname, finputs, outputs);
- _outputHops = outputHops;
- }
-
- public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs)
- {
- super(fnamespace + Program.KEY_DELIM + fname, DataType.UNKNOWN, ValueType.UNKNOWN );
-
- _type = type;
- _fnamespace = fnamespace;
- _fname = fname;
- _outputs = outputs;
-
- for( Hop in : finputs )
- {
- getInput().add(in);
- in.getParent().add(this);
- }
- }
-
- public String getFunctionNamespace()
- {
- return _fnamespace;
- }
-
- public String getFunctionName()
- {
- return _fname;
- }
-
- public void setFunctionName( String fname )
- {
- _fname = fname;
- }
-
- public ArrayList<Hop> getOutputs() {
- return _outputHops;
- }
-
- public String[] getOutputVariableNames()
- {
- return _outputs;
- }
-
- public FunctionType getFunctionType()
- {
- return _type;
- }
-
- @Override
- public boolean allowsAllExecTypes() {
- return false;
- }
-
- @Override
- public void computeMemEstimate( MemoTable memo )
- {
- //overwrites default hops behavior
-
- if( _type == FunctionType.DML )
- _memEstimate = 1; //minimal mem estimate
- else if( _type == FunctionType.EXTERNAL_MEM )
- _memEstimate = 2* getInputSize(); //in/out
- else if( _type == FunctionType.EXTERNAL_FILE || _type == FunctionType.UNKNOWN )
- _memEstimate = CostEstimatorHops.DEFAULT_MEM_MR;
- else if ( _type == FunctionType.MULTIRETURN_BUILTIN ) {
- boolean outputDimsKnown = true;
- for(Hop out : getOutputs()){
- outputDimsKnown &= out.dimsKnown();
- }
- if( outputDimsKnown ) {
- long lnnz = ((_nnz>=0)?_nnz:_dim1*_dim2);
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- _processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
- }
- _memEstimate = getInputOutputSize();
- //System.out.println("QREst " + (_memEstimate/1024/1024));
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
- throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
- else {
- if ( getFunctionName().equalsIgnoreCase("qr") ) {
- // upper-triangular and lower-triangular matrices
- long outputH = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
- long outputR = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
- //System.out.println("QROut " + (outputH+outputR)/1024/1024);
- return outputH+outputR;
-
- }
- else if ( getFunctionName().equalsIgnoreCase("lu") ) {
- // upper-triangular and lower-triangular matrices
- long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0/getOutputs().get(1).getDim2());
- long outputL = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
- long outputU = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
- //System.out.println("LUOut " + (outputL+outputU+outputP)/1024/1024);
- return outputL+outputU+outputP;
-
- }
- else if ( getFunctionName().equalsIgnoreCase("eigen") ) {
- long outputVectors = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
- long outputValues = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), 1, 1.0);
- //System.out.println("EigenOut " + (outputVectors+outputValues)/1024/1024);
- return outputVectors+outputValues;
-
- }
- else
- throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
- }
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
- throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
- else {
- if ( getFunctionName().equalsIgnoreCase("qr") ) {
- // matrix of size same as the input
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
- //System.out.println("QRInter " + interOutput/1024/1024);
- return interOutput;
- }
- else if ( getFunctionName().equalsIgnoreCase("lu")) {
- // 1D vector
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
- //System.out.println("LUInter " + interOutput/1024/1024);
- return interOutput;
- }
- else if ( getFunctionName().equalsIgnoreCase("eigen")) {
- // One matrix of size original input and three 1D vectors (used to represent tridiagonal matrix)
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0)
- + 3*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
- //System.out.println("EigenInter " + interOutput/1024/1024);
- return interOutput;
- }
- else
- throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
- }
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- throw new RuntimeException("Invalid call of inferOutputCharacteristics in FunctionOp.");
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
-
- if ( et != ExecType.CP ) {
- throw new HopsException("Invalid execution type for function: " + _fname);
- }
- //construct input lops (recursive)
- ArrayList<Lop> tmp = new ArrayList<Lop>();
- for( Hop in : getInput() )
- tmp.add( in.constructLops() );
-
- //construct function call
- FunctionCallCP fcall = new FunctionCallCP( tmp, _fnamespace, _fname, _outputs, _outputHops );
- setLineNumbers( fcall );
- setLops( fcall );
-
- //note: no reblock lop because outputs directly bound
-
- return getLops();
- }
-
- @Override
- public String getOpString()
- {
- return OPSTRING;
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- if ( getFunctionType() == FunctionType.MULTIRETURN_BUILTIN ) {
- // Since the memory estimate is only conservative, do not throw
- // exception if the estimated memory is larger than the budget
- // Nevertheless, memory estimates these functions are useful for
- // other purposes, such as compiling parfor
- return ExecType.CP;
-
- // check if there is sufficient memory to execute this function
- /*if ( getMemEstimate() < OptimizerUtils.getMemBudget(true) ) {
- return ExecType.CP;
- }
- else {
- throw new HopsException("Insufficient memory to execute function: " + getFunctionName());
- }*/
- }
- // the actual function call is always CP
- return ExecType.CP;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- //do nothing
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- FunctionOp ret = new FunctionOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._type = _type;
- ret._fnamespace = _fnamespace;
- ret._fname = _fname;
- ret._outputs = _outputs.clone();
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- return false;
- }
-}
[53/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Division.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Division.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Division.dml
index e675d1d..fb5222d 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Division.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Division.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Multiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Multiplication.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Multiplication.dml
index 39a7db7..1dd6b94 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Multiplication.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Multiplication.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Substraction.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Substraction.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Substraction.dml
index a86f8da..ad2528a 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Substraction.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Substraction.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/FullMatrixMultiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/FullMatrixMultiplication.dml b/src/test/scripts/functions/binary/matrix_full_other/FullMatrixMultiplication.dml
index 634bc9e..5ac5465 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/FullMatrixMultiplication.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/FullMatrixMultiplication.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$5, cols=$6, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/FullPower.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/FullPower.dml b/src/test/scripts/functions/binary/matrix_full_other/FullPower.dml
index 78e2cfd..7acd368 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/FullPower.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/FullPower.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1);
B = read($2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_div.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_div.dml b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_div.dml
index dfc87f6..47a616e 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_div.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_div.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullIntegerDivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullIntegerDivisionTest.java
A = read($1);
B = read($2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_mod.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_mod.dml b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_mod.dml
index bd44103..db877d1 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_mod.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/IntegerDivision_mod.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullIntegerDivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullIntegerDivisionTest.java
A = read($1);
B = read($2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication1.dml b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication1.dml
index 03a4715..c8dd635 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication1.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.TransposeSelfMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.TransposeSelfMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = t(A) %*% A;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication2.dml b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication2.dml
index fb90f81..1a3d4d9 100644
--- a/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication2.dml
+++ b/src/test/scripts/functions/binary/matrix_full_other/TransposeSelfMatrixMultiplication2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.TransposeSelfMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.TransposeSelfMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = A %*% t(A);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/AdditionSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/AdditionSingleTest.dml b/src/test/scripts/functions/binary/scalar/AdditionSingleTest.dml
index 978983a..68d1eca 100644
--- a/src/test/scripts/functions/binary/scalar/AdditionSingleTest.dml
+++ b/src/test/scripts/functions/binary/scalar/AdditionSingleTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.AdditionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.AdditionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/AdditionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/AdditionTest.dml b/src/test/scripts/functions/binary/scalar/AdditionTest.dml
index 9d8a1f2..10e3839 100644
--- a/src/test/scripts/functions/binary/scalar/AdditionTest.dml
+++ b/src/test/scripts/functions/binary/scalar/AdditionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.AdditionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.AdditionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/AndTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/AndTest.dml b/src/test/scripts/functions/binary/scalar/AndTest.dml
index c6fcc70..a2304ff 100644
--- a/src/test/scripts/functions/binary/scalar/AndTest.dml
+++ b/src/test/scripts/functions/binary/scalar/AndTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.AndTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.AndTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/DivisionSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/DivisionSingleTest.dml b/src/test/scripts/functions/binary/scalar/DivisionSingleTest.dml
index 5d6daf1..73ee6ad 100644
--- a/src/test/scripts/functions/binary/scalar/DivisionSingleTest.dml
+++ b/src/test/scripts/functions/binary/scalar/DivisionSingleTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.DivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.DivisionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/DivisionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/DivisionTest.dml b/src/test/scripts/functions/binary/scalar/DivisionTest.dml
index b4479bc..362384b 100644
--- a/src/test/scripts/functions/binary/scalar/DivisionTest.dml
+++ b/src/test/scripts/functions/binary/scalar/DivisionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.DivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.DivisionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/EqualTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/EqualTest.dml b/src/test/scripts/functions/binary/scalar/EqualTest.dml
index 700183b..ee5aeee 100644
--- a/src/test/scripts/functions/binary/scalar/EqualTest.dml
+++ b/src/test/scripts/functions/binary/scalar/EqualTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.EqualTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.EqualTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/GreaterThanOrEqualTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/GreaterThanOrEqualTest.dml b/src/test/scripts/functions/binary/scalar/GreaterThanOrEqualTest.dml
index b77b96b..baab402 100644
--- a/src/test/scripts/functions/binary/scalar/GreaterThanOrEqualTest.dml
+++ b/src/test/scripts/functions/binary/scalar/GreaterThanOrEqualTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.GreaterThanOrEqualTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.GreaterThanOrEqualTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/GreaterThanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/GreaterThanTest.dml b/src/test/scripts/functions/binary/scalar/GreaterThanTest.dml
index 66194e6..957b642 100644
--- a/src/test/scripts/functions/binary/scalar/GreaterThanTest.dml
+++ b/src/test/scripts/functions/binary/scalar/GreaterThanTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.GreaterThanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.GreaterThanTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/LessThanOrEqualTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/LessThanOrEqualTest.dml b/src/test/scripts/functions/binary/scalar/LessThanOrEqualTest.dml
index ea8e97a..69a43fd 100644
--- a/src/test/scripts/functions/binary/scalar/LessThanOrEqualTest.dml
+++ b/src/test/scripts/functions/binary/scalar/LessThanOrEqualTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.LessThanOrEqualTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.LessThanOrEqualTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/LessThanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/LessThanTest.dml b/src/test/scripts/functions/binary/scalar/LessThanTest.dml
index f371d79..59d68dc 100644
--- a/src/test/scripts/functions/binary/scalar/LessThanTest.dml
+++ b/src/test/scripts/functions/binary/scalar/LessThanTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.LessThanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.LessThanTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/LogarithmTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/LogarithmTest.dml b/src/test/scripts/functions/binary/scalar/LogarithmTest.dml
index 19c5644..18b7029 100644
--- a/src/test/scripts/functions/binary/scalar/LogarithmTest.dml
+++ b/src/test/scripts/functions/binary/scalar/LogarithmTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.LogarithmTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.LogarithmTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/ModulusSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/ModulusSingleTest.dml b/src/test/scripts/functions/binary/scalar/ModulusSingleTest.dml
index a1d3d8a..9af6ac2 100644
--- a/src/test/scripts/functions/binary/scalar/ModulusSingleTest.dml
+++ b/src/test/scripts/functions/binary/scalar/ModulusSingleTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.ModulusTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.ModulusTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/ModulusTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/ModulusTest.dml b/src/test/scripts/functions/binary/scalar/ModulusTest.dml
index a4ffc4f..0bbf81f 100644
--- a/src/test/scripts/functions/binary/scalar/ModulusTest.dml
+++ b/src/test/scripts/functions/binary/scalar/ModulusTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.ModulusTest
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.ModulusTest
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/MultiplicationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/MultiplicationTest.dml b/src/test/scripts/functions/binary/scalar/MultiplicationTest.dml
index 38fd95c..4c06fc7 100644
--- a/src/test/scripts/functions/binary/scalar/MultiplicationTest.dml
+++ b/src/test/scripts/functions/binary/scalar/MultiplicationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.MultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.MultiplicationTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/OrTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/OrTest.dml b/src/test/scripts/functions/binary/scalar/OrTest.dml
index d050c22..78f9639 100644
--- a/src/test/scripts/functions/binary/scalar/OrTest.dml
+++ b/src/test/scripts/functions/binary/scalar/OrTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.OrTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.OrTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/PowerTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/PowerTest.dml b/src/test/scripts/functions/binary/scalar/PowerTest.dml
index a72bd54..772b89c 100644
--- a/src/test/scripts/functions/binary/scalar/PowerTest.dml
+++ b/src/test/scripts/functions/binary/scalar/PowerTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.MultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.MultiplicationTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/SubtractionMultipleOperantsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/SubtractionMultipleOperantsTest.dml b/src/test/scripts/functions/binary/scalar/SubtractionMultipleOperantsTest.dml
index 4643fca..4d74d10 100644
--- a/src/test/scripts/functions/binary/scalar/SubtractionMultipleOperantsTest.dml
+++ b/src/test/scripts/functions/binary/scalar/SubtractionMultipleOperantsTest.dml
@@ -17,7 +17,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.SubtractionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.SubtractionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/scalar/SubtractionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/scalar/SubtractionTest.dml b/src/test/scripts/functions/binary/scalar/SubtractionTest.dml
index 4dbe08a..4728024 100644
--- a/src/test/scripts/functions/binary/scalar/SubtractionTest.dml
+++ b/src/test/scripts/functions/binary/scalar/SubtractionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.scalar.SubtractionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.scalar.SubtractionTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/blocks/VariableAnalysis1Test.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/blocks/VariableAnalysis1Test.dml b/src/test/scripts/functions/blocks/VariableAnalysis1Test.dml
index b77b54c..a195231 100644
--- a/src/test/scripts/functions/blocks/VariableAnalysis1Test.dml
+++ b/src/test/scripts/functions/blocks/VariableAnalysis1Test.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# JUnit test class: com.ibm.bi.dml.test.integration.functions.blocks.VariableTest.java
+# JUnit test class: org.apache.sysml.test.integration.functions.blocks.VariableTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/blocks/VariablePassing1Test.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/blocks/VariablePassing1Test.dml b/src/test/scripts/functions/blocks/VariablePassing1Test.dml
index 93f5734..6dab770 100644
--- a/src/test/scripts/functions/blocks/VariablePassing1Test.dml
+++ b/src/test/scripts/functions/blocks/VariablePassing1Test.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# JUnit test class: com.ibm.bi.dml.test.integration.functions.blocks.VariableTest.java
+# JUnit test class: org.apache.sysml.test.integration.functions.blocks.VariableTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/blocks/VariablePassing2Test.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/blocks/VariablePassing2Test.dml b/src/test/scripts/functions/blocks/VariablePassing2Test.dml
index 323c4a4..0c267fa 100644
--- a/src/test/scripts/functions/blocks/VariablePassing2Test.dml
+++ b/src/test/scripts/functions/blocks/VariablePassing2Test.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# JUnit test class: com.ibm.bi.dml.test.integration.functions.blocks.VariableTest.jav
+# JUnit test class: org.apache.sysml.test.integration.functions.blocks.VariableTest.jav
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = $$factor$$ * t(A);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/blocks/WhileTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/blocks/WhileTest.dml b/src/test/scripts/functions/blocks/WhileTest.dml
index 5a83a88..34f8a9b 100644
--- a/src/test/scripts/functions/blocks/WhileTest.dml
+++ b/src/test/scripts/functions/blocks/WhileTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# JUnit test class: com.ibm.bi.dml.test.integration.functions.blocks.WhileTest.java
+# JUnit test class: org.apache.sysml.test.integration.functions.blocks.WhileTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
i = 0;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/CopyVariableTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/CopyVariableTest.dml b/src/test/scripts/functions/data/CopyVariableTest.dml
index d59d1dc..e512c13 100644
--- a/src/test/scripts/functions/data/CopyVariableTest.dml
+++ b/src/test/scripts/functions/data/CopyVariableTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.VariableTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.VariableTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = A;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/RandScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/RandScalarTest.dml b/src/test/scripts/functions/data/RandScalarTest.dml
index f7a8aeb..7500690 100644
--- a/src/test/scripts/functions/data/RandScalarTest.dml
+++ b/src/test/scripts/functions/data/RandScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.RandTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.RandTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/RandTest1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/RandTest1.dml b/src/test/scripts/functions/data/RandTest1.dml
index 4753ca0..3664300 100644
--- a/src/test/scripts/functions/data/RandTest1.dml
+++ b/src/test/scripts/functions/data/RandTest1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.RandTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.RandTest.java
A = Rand(rows=$$rows$$, cols=$$cols$$, min=$$min$$, max=$$max$$);
write(A, "$$outdir$$rand", format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/RandTest2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/RandTest2.dml b/src/test/scripts/functions/data/RandTest2.dml
index fbbc818..6a456a9 100644
--- a/src/test/scripts/functions/data/RandTest2.dml
+++ b/src/test/scripts/functions/data/RandTest2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.RandTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.RandTest.java
B = Rand(rows=$$rows$$*5, cols=$$cols$$, min=$$min$$, max=$$max$$);
write(B, "$$outdir$$rand", format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/RandTest3.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/RandTest3.dml b/src/test/scripts/functions/data/RandTest3.dml
index 5f207b0..85a2ee4 100644
--- a/src/test/scripts/functions/data/RandTest3.dml
+++ b/src/test/scripts/functions/data/RandTest3.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.RandTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.RandTest.java
C = Rand(rows=$$rows$$+ 0.5, cols=$$cols$$, min=$$min$$, max=$$max$$);
write(C, "$$outdir$$rand", format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/RandTest4.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/RandTest4.dml b/src/test/scripts/functions/data/RandTest4.dml
index 253e897..97d7a1e 100644
--- a/src/test/scripts/functions/data/RandTest4.dml
+++ b/src/test/scripts/functions/data/RandTest4.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.RandTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.RandTest.java
A = read("$$indir$$"+"a", rows=$$rows$$, cols=$$cols$$, format="$$format$$");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/ReadMMIndexTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/ReadMMIndexTest.dml b/src/test/scripts/functions/data/ReadMMIndexTest.dml
index e4ea3a9..75c2e88 100644
--- a/src/test/scripts/functions/data/ReadMMIndexTest.dml
+++ b/src/test/scripts/functions/data/ReadMMIndexTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.readTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.readTest.java
A = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="$$format$$"$$blockrows$$$$blockcols$$);
write(A, "$$outdir$$b", format="text");
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/ReadMMTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/ReadMMTest.dml b/src/test/scripts/functions/data/ReadMMTest.dml
index 4a37349..65225a3 100644
--- a/src/test/scripts/functions/data/ReadMMTest.dml
+++ b/src/test/scripts/functions/data/ReadMMTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.readTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.readTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="$$format$$");
write(A, "$$outdir$$a", format="text");
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/ReblockTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/ReblockTest.dml b/src/test/scripts/functions/data/ReblockTest.dml
index 39e8365..4b61dd8 100644
--- a/src/test/scripts/functions/data/ReblockTest.dml
+++ b/src/test/scripts/functions/data/ReblockTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.ReblockTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.ReblockTest.java
A = read("$$indir$$a", rows=10, cols=10);
write(A, "$$outdir$$a", format="text");
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/WriteMMComplexTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/WriteMMComplexTest.dml b/src/test/scripts/functions/data/WriteMMComplexTest.dml
index 07bb468..cbce714 100644
--- a/src/test/scripts/functions/data/WriteMMComplexTest.dml
+++ b/src/test/scripts/functions/data/WriteMMComplexTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.WriteMMTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.WriteMMTest.java
A = read ($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/WriteMMTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/WriteMMTest.dml b/src/test/scripts/functions/data/WriteMMTest.dml
index 08a7d3f..4c649c0 100644
--- a/src/test/scripts/functions/data/WriteMMTest.dml
+++ b/src/test/scripts/functions/data/WriteMMTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.WriteMMTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.WriteMMTest.java
A = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/WriteTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/WriteTest.dml b/src/test/scripts/functions/data/WriteTest.dml
index e2af861..0736d51 100644
--- a/src/test/scripts/functions/data/WriteTest.dml
+++ b/src/test/scripts/functions/data/WriteTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.writeTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.writeTest.java
A = read("$$indir$$a");
write(A, "$$outdir$$a", format="$$format$$");
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/data/WriteTwiceTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/data/WriteTwiceTest.dml b/src/test/scripts/functions/data/WriteTwiceTest.dml
index bcf6bbe..04fb113 100644
--- a/src/test/scripts/functions/data/WriteTwiceTest.dml
+++ b/src/test/scripts/functions/data/WriteTwiceTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.data.writeTest.java
+# junit test class: org.apache.sysml.test.integration.functions.data.writeTest.java
A = read("$$indir$$a");
write(A, "$$outdir$$b", format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/dmlscript/DMLScriptTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/dmlscript/DMLScriptTest.dml b/src/test/scripts/functions/dmlscript/DMLScriptTest.dml
index 01aaf36..a3ff553 100644
--- a/src/test/scripts/functions/dmlscript/DMLScriptTest.dml
+++ b/src/test/scripts/functions/dmlscript/DMLScriptTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.dmlscript.DMLScriptTest.java
+# junit test class: org.apache.sysml.test.integration.functions.dmlscript.DMLScriptTest.java
A = read($1, rows=$2, cols=$3, format=$4);
write(A, $5, format=$4);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/DynProject.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/DynProject.dml b/src/test/scripts/functions/external/DynProject.dml
index 368d284..9390df5 100644
--- a/src/test/scripts/functions/external/DynProject.dml
+++ b/src/test/scripts/functions/external/DynProject.dml
@@ -19,7 +19,7 @@
dynProject = externalFunction(Matrix[Double] B, Matrix[Double] c)
return (Matrix[Double] PD)
- implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicProjectMatrixCP",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.DynamicProjectMatrixCP",exectype="mem")
X = read($1, rows=$3, cols=$4, format="text");
c = read($2, rows=1, cols=$5, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/DynReadWrite.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/DynReadWrite.dml b/src/test/scripts/functions/external/DynReadWrite.dml
index 2164db0..d8916aa 100644
--- a/src/test/scripts/functions/external/DynReadWrite.dml
+++ b/src/test/scripts/functions/external/DynReadWrite.dml
@@ -19,11 +19,11 @@
dynRead = externalFunction(String fname, Integer rows, Integer cols, String format)
return (Matrix[Double] M)
-implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicReadMatrixCP",exectype="mem")
+implemented in (classname="org.apache.sysml.udf.lib.DynamicReadMatrixCP",exectype="mem")
dynWrite = externalFunction(Matrix[Double] input, String fname, String format)
return(Boolean success)
-implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
+implemented in (classname="org.apache.sysml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
X = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/FunctionExpressions1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/FunctionExpressions1.dml b/src/test/scripts/functions/external/FunctionExpressions1.dml
index 7f97bb3..7d48c48 100644
--- a/src/test/scripts/functions/external/FunctionExpressions1.dml
+++ b/src/test/scripts/functions/external/FunctionExpressions1.dml
@@ -18,7 +18,7 @@
orderExternal = externalFunction(Matrix[Double] A, Integer col, Boolean desc) return (Matrix[Double] B)
- implemented in (classname="com.ibm.bi.dml.udf.lib.OrderWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.OrderWrapper",exectype="mem")
foo = function( Matrix[Double] A ) return (Matrix[Double] B)
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/FunctionExpressions2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/FunctionExpressions2.dml b/src/test/scripts/functions/external/FunctionExpressions2.dml
index 6add763..dce8dd8 100644
--- a/src/test/scripts/functions/external/FunctionExpressions2.dml
+++ b/src/test/scripts/functions/external/FunctionExpressions2.dml
@@ -18,7 +18,7 @@
orderExternal = externalFunction(Matrix[Double] A, Integer col, Boolean desc) return (Matrix[Double] B)
- implemented in (classname="com.ibm.bi.dml.udf.lib.OrderWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.OrderWrapper",exectype="mem")
foo = function( Matrix[Double] A ) return (Matrix[Double] B)
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/Order1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/Order1.dml b/src/test/scripts/functions/external/Order1.dml
index 13995fa..b29166b 100644
--- a/src/test/scripts/functions/external/Order1.dml
+++ b/src/test/scripts/functions/external/Order1.dml
@@ -18,7 +18,7 @@
orderExternal = externalFunction(Matrix[Double] A, Integer col, Boolean desc) return (Matrix[Double] B)
- implemented in (classname="com.ibm.bi.dml.udf.lib.OrderWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.OrderWrapper",exectype="mem")
A = read( $1, rows=$2, cols=$3);
B1 = orderExternal(A, $4, FALSE);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/Order2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/Order2.dml b/src/test/scripts/functions/external/Order2.dml
index 3db877d..3da974e 100644
--- a/src/test/scripts/functions/external/Order2.dml
+++ b/src/test/scripts/functions/external/Order2.dml
@@ -18,7 +18,7 @@
orderExternal = externalFunction(Matrix[Double] A, Integer col, Boolean desc) return (Matrix[Double] B)
- implemented in (classname="com.ibm.bi.dml.udf.lib.OrderWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.OrderWrapper",exectype="mem")
A = read( $1, rows=$2, cols=$3);
B1 = orderExternal(A, $4, TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/Outlier.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/Outlier.dml b/src/test/scripts/functions/external/Outlier.dml
index 437f6a2..377e7f0 100644
--- a/src/test/scripts/functions/external/Outlier.dml
+++ b/src/test/scripts/functions/external/Outlier.dml
@@ -26,7 +26,7 @@
outliers = externalFunction(Matrix[Double] D, Integer m, Integer k)
return (Matrix[Double] C)
-implemented in (classname="com.ibm.bi.dml.udf.lib.OutlierWrapper",execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.OutlierWrapper",execlocation="master")
M = read( $1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/SGDMFTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/SGDMFTest.dml b/src/test/scripts/functions/external/SGDMFTest.dml
index 4f1a0e0..d0d5c50 100644
--- a/src/test/scripts/functions/external/SGDMFTest.dml
+++ b/src/test/scripts/functions/external/SGDMFTest.dml
@@ -18,7 +18,7 @@
sgd = externalFunction(Matrix[Double] V, Integer n, Integer m, Integer e, Integer factors)
return (Matrix[Double] W, Matrix[Double] tH)
-implemented in (classname="com.ibm.bi.dml.udf.lib.SGDWrapper", execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.SGDWrapper", execlocation="master")
#n X m is the number of blocks the ratings matrix is divided into
#each block is of size rows(V)/n X cols(V)/m cells
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/SeqMiner.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/SeqMiner.dml b/src/test/scripts/functions/external/SeqMiner.dml
index b05ead6..f6b2ea1 100644
--- a/src/test/scripts/functions/external/SeqMiner.dml
+++ b/src/test/scripts/functions/external/SeqMiner.dml
@@ -25,7 +25,7 @@
seqMini = externalFunction(Matrix[Double] A, Double sup, Double max_level)
return (Matrix[Double] B, Matrix[Double] C)
-implemented in (classname="com.ibm.bi.dml.udf.lib.SequenceMinerWrapper",execlocation="master");
+implemented in (classname="org.apache.sysml.udf.lib.SequenceMinerWrapper",execlocation="master");
minsup = 0.001;
max_level=5.0;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/kMeans.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/kMeans.dml b/src/test/scripts/functions/external/kMeans.dml
index 91c9229..78c1104 100644
--- a/src/test/scripts/functions/external/kMeans.dml
+++ b/src/test/scripts/functions/external/kMeans.dml
@@ -26,7 +26,7 @@
kMeans = externalFunction(Matrix[Double] D, Integer k)
return (Matrix[Double] C)
-implemented in (classname="com.ibm.bi.dml.udf.lib.kMeansWrapper",execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.kMeansWrapper",execlocation="master")
M = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/external/kMeans2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/external/kMeans2.dml b/src/test/scripts/functions/external/kMeans2.dml
index 63cb9b7..2c1a9e1 100644
--- a/src/test/scripts/functions/external/kMeans2.dml
+++ b/src/test/scripts/functions/external/kMeans2.dml
@@ -26,11 +26,11 @@
kMeans = externalFunction(Matrix[Double] D, Integer k)
return (Matrix[Double] C)
-implemented in (classname="com.ibm.bi.dml.udf.lib.kMeansWrapper",execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.kMeansWrapper",execlocation="master")
kMeansWithInit = externalFunction(Matrix[Double] D, Integer k, Matrix[Double] initK)
return (Matrix[Double] C)
-implemented in (classname="com.ibm.bi.dml.udf.lib.kMeansWrapper",execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.kMeansWrapper",execlocation="master")
M = read($1, rows=$2, cols=$3, format="text");
initialCenters = read($4, rows=5, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/io/IOTest1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/io/IOTest1.dml b/src/test/scripts/functions/io/IOTest1.dml
index 88dfbd8..821462f 100644
--- a/src/test/scripts/functions/io/IOTest1.dml
+++ b/src/test/scripts/functions/io/IOTest1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.io.IOTest1.java
+# junit test class: org.apache.sysml.test.integration.functions.io.IOTest1.java
A = read("$$indir$$"+"a", rows=$$rows$$, cols=$$cols$$, format="$$format$$");
write(A, "$$outdir$$"+"a", format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/io/IOTest2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/io/IOTest2.dml b/src/test/scripts/functions/io/IOTest2.dml
index 210db41..059402e 100644
--- a/src/test/scripts/functions/io/IOTest2.dml
+++ b/src/test/scripts/functions/io/IOTest2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.io.IOTest2.java
+# junit test class: org.apache.sysml.test.integration.functions.io.IOTest2.java
A = read("$$indir$$"+a, rows=$$rows$$, cols=$$cols$$, format="$$format$$");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/io/IOTest3.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/io/IOTest3.dml b/src/test/scripts/functions/io/IOTest3.dml
index 98e2fec..1d82119 100644
--- a/src/test/scripts/functions/io/IOTest3.dml
+++ b/src/test/scripts/functions/io/IOTest3.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.io.IOTest3.java
+# junit test class: org.apache.sysml.test.integration.functions.io.IOTest3.java
A = read("$$indir$$"+"a", rows=$$rows$$, cols=$$cols$$, format="$$format$$");
write(A, "$$outdir$$"+a, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/io/IOTest4.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/io/IOTest4.dml b/src/test/scripts/functions/io/IOTest4.dml
index e806b53..e7d534a 100644
--- a/src/test/scripts/functions/io/IOTest4.dml
+++ b/src/test/scripts/functions/io/IOTest4.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.io.IOTest4.java
+# junit test class: org.apache.sysml.test.integration.functions.io.IOTest4.java
B = 12;
if( 1==1 ){
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/io/IOTest5.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/io/IOTest5.dml b/src/test/scripts/functions/io/IOTest5.dml
index 0856cae..0c767ba 100644
--- a/src/test/scripts/functions/io/IOTest5.dml
+++ b/src/test/scripts/functions/io/IOTest5.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.io.IOTest5.java
+# junit test class: org.apache.sysml.test.integration.functions.io.IOTest5.java
B = 12;
if( 1==1 ){
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor50.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor50.dml b/src/test/scripts/functions/parfor/parfor50.dml
index 76ba152..c15b9a4 100644
--- a/src/test/scripts/functions/parfor/parfor50.dml
+++ b/src/test/scripts/functions/parfor/parfor50.dml
@@ -18,7 +18,7 @@
binning = externalFunction(Matrix[Double] A, Integer binsize, Integer numbins) return (Matrix[Double] B, Integer numbinsdef)
- implemented in (classname="com.ibm.bi.dml.udf.lib.BinningWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.BinningWrapper",exectype="mem")
fmt = ifdef($fmt, "text");
num_bins = ifdef($bins, 3); # 50
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor_extfunct.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor_extfunct.dml b/src/test/scripts/functions/parfor/parfor_extfunct.dml
index 6e9bc74..ff65eeb 100644
--- a/src/test/scripts/functions/parfor/parfor_extfunct.dml
+++ b/src/test/scripts/functions/parfor/parfor_extfunct.dml
@@ -19,11 +19,11 @@
dynRead = externalFunction(String fname, Integer rows, Integer cols, String format)
return (Matrix[Double] M)
-implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicReadMatrixCP",exectype="mem")
+implemented in (classname="org.apache.sysml.udf.lib.DynamicReadMatrixCP",exectype="mem")
execCondense = externalFunction(Matrix[Double] input)
return(Matrix[Double] out)
-implemented in (classname="com.ibm.bi.dml.udf.lib.RemoveEmptyRows",exectype="file", execlocation="master")
+implemented in (classname="org.apache.sysml.udf.lib.RemoveEmptyRows",exectype="file", execlocation="master")
V = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor_literals1a.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor_literals1a.dml b/src/test/scripts/functions/parfor/parfor_literals1a.dml
index 5ca3c71..2ae7d50 100644
--- a/src/test/scripts/functions/parfor/parfor_literals1a.dml
+++ b/src/test/scripts/functions/parfor/parfor_literals1a.dml
@@ -19,7 +19,7 @@
dynWrite = externalFunction(Matrix[Double] input, String fname, String format)
return(Boolean success)
- implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
A = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor_literals1b.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor_literals1b.dml b/src/test/scripts/functions/parfor/parfor_literals1b.dml
index dfb79ca..c4b8027 100644
--- a/src/test/scripts/functions/parfor/parfor_literals1b.dml
+++ b/src/test/scripts/functions/parfor/parfor_literals1b.dml
@@ -19,7 +19,7 @@
dynWrite = externalFunction(Matrix[Double] input, String fname, String format)
return(Boolean success)
- implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
A = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor_literals1c.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor_literals1c.dml b/src/test/scripts/functions/parfor/parfor_literals1c.dml
index fd107c5..86adfd9 100644
--- a/src/test/scripts/functions/parfor/parfor_literals1c.dml
+++ b/src/test/scripts/functions/parfor/parfor_literals1c.dml
@@ -19,7 +19,7 @@
dynWrite = externalFunction(Matrix[Double] input, String fname, String format)
return(Boolean success)
- implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
A = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/parfor/parfor_literals2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor_literals2.dml b/src/test/scripts/functions/parfor/parfor_literals2.dml
index d782e83..bb6d722 100644
--- a/src/test/scripts/functions/parfor/parfor_literals2.dml
+++ b/src/test/scripts/functions/parfor/parfor_literals2.dml
@@ -19,7 +19,7 @@
dynWrite_t0 = externalFunction(Matrix[Double] input, String fname, String format)
return(Boolean success)
- implemented in (classname="com.ibm.bi.dml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.DynamicWriteMatrixCP",exectype="mem")
A = read($1, rows=$2, cols=$3, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/terms/TestScalarToMatrixInLoop.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/terms/TestScalarToMatrixInLoop.dml b/src/test/scripts/functions/terms/TestScalarToMatrixInLoop.dml
index 43e48c9..8a51538 100644
--- a/src/test/scripts/functions/terms/TestScalarToMatrixInLoop.dml
+++ b/src/test/scripts/functions/terms/TestScalarToMatrixInLoop.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.terms.ScalarToMatrixInLoop.java
+# junit test class: org.apache.sysml.test.integration.functions.terms.ScalarToMatrixInLoop.java
A = Rand(cols=$$cols$$, rows=$$rows$$);
W = 5;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/terms/TestTerm1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/terms/TestTerm1.dml b/src/test/scripts/functions/terms/TestTerm1.dml
index 51164a3..3d9c221 100644
--- a/src/test/scripts/functions/terms/TestTerm1.dml
+++ b/src/test/scripts/functions/terms/TestTerm1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.terms.ScalarMatrixUnaryBinaryTest.java
+# junit test class: org.apache.sysml.test.integration.functions.terms.ScalarMatrixUnaryBinaryTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/ACosTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/ACosTest.dml b/src/test/scripts/functions/unary/matrix/ACosTest.dml
index 7ec93db..f22b592 100644
--- a/src/test/scripts/functions/unary/matrix/ACosTest.dml
+++ b/src/test/scripts/functions/unary/matrix/ACosTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.CosTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.CosTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
CosVector = acos(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/ASinTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/ASinTest.dml b/src/test/scripts/functions/unary/matrix/ASinTest.dml
index 3d29ec0..e33e11a 100644
--- a/src/test/scripts/functions/unary/matrix/ASinTest.dml
+++ b/src/test/scripts/functions/unary/matrix/ASinTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.SinTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.SinTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
SinVector = asin(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/ATanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/ATanTest.dml b/src/test/scripts/functions/unary/matrix/ATanTest.dml
index 2fcf1f5..b2c9e89 100644
--- a/src/test/scripts/functions/unary/matrix/ATanTest.dml
+++ b/src/test/scripts/functions/unary/matrix/ATanTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.TanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.TanTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
TanVector = atan(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/AbsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/AbsTest.dml b/src/test/scripts/functions/unary/matrix/AbsTest.dml
index 17703e9..87b9739 100644
--- a/src/test/scripts/functions/unary/matrix/AbsTest.dml
+++ b/src/test/scripts/functions/unary/matrix/AbsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.AbsTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.AbsTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
AbsVector = abs(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/CastAsScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/CastAsScalarTest.dml b/src/test/scripts/functions/unary/matrix/CastAsScalarTest.dml
index 3931e4e..714332a 100644
--- a/src/test/scripts/functions/unary/matrix/CastAsScalarTest.dml
+++ b/src/test/scripts/functions/unary/matrix/CastAsScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.CastAsScalarTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.CastAsScalarTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/CosTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/CosTest.dml b/src/test/scripts/functions/unary/matrix/CosTest.dml
index dfd25be..d498e10 100644
--- a/src/test/scripts/functions/unary/matrix/CosTest.dml
+++ b/src/test/scripts/functions/unary/matrix/CosTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.CosTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.CosTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
CosVector = cos(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/DiagSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/DiagSingleTest.dml b/src/test/scripts/functions/unary/matrix/DiagSingleTest.dml
index 4519787..e3aaec1 100644
--- a/src/test/scripts/functions/unary/matrix/DiagSingleTest.dml
+++ b/src/test/scripts/functions/unary/matrix/DiagSingleTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.DiagTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.DiagTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = diag(A);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/DiagTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/DiagTest.dml b/src/test/scripts/functions/unary/matrix/DiagTest.dml
index 60a9a2b..9e58c52 100644
--- a/src/test/scripts/functions/unary/matrix/DiagTest.dml
+++ b/src/test/scripts/functions/unary/matrix/DiagTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.DiagTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.DiagTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = diag(A);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/NegationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/NegationTest.dml b/src/test/scripts/functions/unary/matrix/NegationTest.dml
index 7fbbac7..f918f73 100644
--- a/src/test/scripts/functions/unary/matrix/NegationTest.dml
+++ b/src/test/scripts/functions/unary/matrix/NegationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.NegationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.NegationTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
NegativeVector = -Vector
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/QRsolve.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/QRsolve.dml b/src/test/scripts/functions/unary/matrix/QRsolve.dml
index 3752be0..7ed0265 100644
--- a/src/test/scripts/functions/unary/matrix/QRsolve.dml
+++ b/src/test/scripts/functions/unary/matrix/QRsolve.dml
@@ -2,7 +2,7 @@
# Ax = b: solve for x via QR decomposition
#solve = externalFunction(Matrix[Double] a, Matrix[Double] b) return(Matrix[Double] c)
-# implemented in (classname="com.ibm.bi.dml.udf.lib.LinearSolverWrapperCP",exectype="mem")
+# implemented in (classname="org.apache.sysml.udf.lib.LinearSolverWrapperCP",exectype="mem")
A = read($1);
b = read($2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/SinTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/SinTest.dml b/src/test/scripts/functions/unary/matrix/SinTest.dml
index 9eb3d34..61195dc 100644
--- a/src/test/scripts/functions/unary/matrix/SinTest.dml
+++ b/src/test/scripts/functions/unary/matrix/SinTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.SinTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.SinTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
SinVector = sin(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/SqrtSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/SqrtSingleTest.dml b/src/test/scripts/functions/unary/matrix/SqrtSingleTest.dml
index 9095dd8..ef6ae40 100644
--- a/src/test/scripts/functions/unary/matrix/SqrtSingleTest.dml
+++ b/src/test/scripts/functions/unary/matrix/SqrtSingleTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.SqrtTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.SqrtTest.java
In = read("$$indir$$in", rows=$$rows$$, cols=$$cols$$, format="text");
Out = sqrt(In);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/SqrtTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/SqrtTest.dml b/src/test/scripts/functions/unary/matrix/SqrtTest.dml
index 2c47720..93ebb03 100644
--- a/src/test/scripts/functions/unary/matrix/SqrtTest.dml
+++ b/src/test/scripts/functions/unary/matrix/SqrtTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.SqrtTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.SqrtTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
SqrtVector = sqrt(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/TanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/TanTest.dml b/src/test/scripts/functions/unary/matrix/TanTest.dml
index d0dbc95..aa7915c 100644
--- a/src/test/scripts/functions/unary/matrix/TanTest.dml
+++ b/src/test/scripts/functions/unary/matrix/TanTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.TanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.TanTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
TanVector = tan(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/matrix/TransposeTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/TransposeTest.dml b/src/test/scripts/functions/unary/matrix/TransposeTest.dml
index 0202d68..a05281f 100644
--- a/src/test/scripts/functions/unary/matrix/TransposeTest.dml
+++ b/src/test/scripts/functions/unary/matrix/TransposeTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.matrix.TransposeTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.matrix.TransposeTest.java
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
TransposedVector = t(Vector);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/ACosTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/ACosTest.dml b/src/test/scripts/functions/unary/scalar/ACosTest.dml
index 26704e6..27b0652 100644
--- a/src/test/scripts/functions/unary/scalar/ACosTest.dml
+++ b/src/test/scripts/functions/unary/scalar/ACosTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.CosTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.CosTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/ASinTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/ASinTest.dml b/src/test/scripts/functions/unary/scalar/ASinTest.dml
index fd80025..a102bb0 100644
--- a/src/test/scripts/functions/unary/scalar/ASinTest.dml
+++ b/src/test/scripts/functions/unary/scalar/ASinTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.SinTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.SinTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/ATanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/ATanTest.dml b/src/test/scripts/functions/unary/scalar/ATanTest.dml
index 6739f46..84468f2 100644
--- a/src/test/scripts/functions/unary/scalar/ATanTest.dml
+++ b/src/test/scripts/functions/unary/scalar/ATanTest.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.TanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.TanTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/AbsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/AbsTest.dml b/src/test/scripts/functions/unary/scalar/AbsTest.dml
index fce1036..b360f1c 100644
--- a/src/test/scripts/functions/unary/scalar/AbsTest.dml
+++ b/src/test/scripts/functions/unary/scalar/AbsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.AbsTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.AbsTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/BooleanWhileTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/BooleanWhileTest.dml b/src/test/scripts/functions/unary/scalar/BooleanWhileTest.dml
index 23baa6e..f7748ea 100644
--- a/src/test/scripts/functions/unary/scalar/BooleanWhileTest.dml
+++ b/src/test/scripts/functions/unary/scalar/BooleanWhileTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.BooleanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.BooleanTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/CosTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/CosTest.dml b/src/test/scripts/functions/unary/scalar/CosTest.dml
index 39e77ff..ce4111a 100644
--- a/src/test/scripts/functions/unary/scalar/CosTest.dml
+++ b/src/test/scripts/functions/unary/scalar/CosTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.CosTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.CosTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/ExponentBinaryTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/ExponentBinaryTest.dml b/src/test/scripts/functions/unary/scalar/ExponentBinaryTest.dml
index f66c4d8..fcf73b3 100644
--- a/src/test/scripts/functions/unary/scalar/ExponentBinaryTest.dml
+++ b/src/test/scripts/functions/unary/scalar/ExponentBinaryTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.ExponentTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.ExponentTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/ExponentTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/ExponentTest.dml b/src/test/scripts/functions/unary/scalar/ExponentTest.dml
index 6397351..55e08ed 100644
--- a/src/test/scripts/functions/unary/scalar/ExponentTest.dml
+++ b/src/test/scripts/functions/unary/scalar/ExponentTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.ExponentTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.ExponentTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/NegationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/NegationTest.dml b/src/test/scripts/functions/unary/scalar/NegationTest.dml
index 2d56203..faa769d 100644
--- a/src/test/scripts/functions/unary/scalar/NegationTest.dml
+++ b/src/test/scripts/functions/unary/scalar/NegationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.NegationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.NegationTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/NotTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/NotTest.dml b/src/test/scripts/functions/unary/scalar/NotTest.dml
index 8aed324..c8c8fcf 100644
--- a/src/test/scripts/functions/unary/scalar/NotTest.dml
+++ b/src/test/scripts/functions/unary/scalar/NotTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.NegationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.NegationTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/PrintTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/PrintTest.dml b/src/test/scripts/functions/unary/scalar/PrintTest.dml
index 3f91d28..35a7f35 100644
--- a/src/test/scripts/functions/unary/scalar/PrintTest.dml
+++ b/src/test/scripts/functions/unary/scalar/PrintTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.PrintTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.PrintTest.java
x=$$value$$;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/PrintTest2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/PrintTest2.dml b/src/test/scripts/functions/unary/scalar/PrintTest2.dml
index 3f6bdbf..26144b6 100644
--- a/src/test/scripts/functions/unary/scalar/PrintTest2.dml
+++ b/src/test/scripts/functions/unary/scalar/PrintTest2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.PrintTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.PrintTest.java
x=$$value$$;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/SinTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/SinTest.dml b/src/test/scripts/functions/unary/scalar/SinTest.dml
index 1e87226..9a902ff 100644
--- a/src/test/scripts/functions/unary/scalar/SinTest.dml
+++ b/src/test/scripts/functions/unary/scalar/SinTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.SinTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.SinTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/SqrtSingleTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/SqrtSingleTest.dml b/src/test/scripts/functions/unary/scalar/SqrtSingleTest.dml
index 7ea34f0..0f2ea98 100644
--- a/src/test/scripts/functions/unary/scalar/SqrtSingleTest.dml
+++ b/src/test/scripts/functions/unary/scalar/SqrtSingleTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.SqrtTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.SqrtTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/SqrtTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/SqrtTest.dml b/src/test/scripts/functions/unary/scalar/SqrtTest.dml
index 2b96064..40854d4 100644
--- a/src/test/scripts/functions/unary/scalar/SqrtTest.dml
+++ b/src/test/scripts/functions/unary/scalar/SqrtTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.SqrtTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.SqrtTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTest1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTest1.dml b/src/test/scripts/functions/unary/scalar/StopTest1.dml
index 55827a1..7a9a7b2 100644
--- a/src/test/scripts/functions/unary/scalar/StopTest1.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTest1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
val = $1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTest2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTest2.dml b/src/test/scripts/functions/unary/scalar/StopTest2.dml
index be2f339..605c813 100644
--- a/src/test/scripts/functions/unary/scalar/StopTest2.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTest2.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
A = matrix(1, rows=10, cols=1);
print(sum(A));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTestLoops_fn.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTestLoops_fn.dml b/src/test/scripts/functions/unary/scalar/StopTestLoops_fn.dml
index bf5366e..f94a02e 100644
--- a/src/test/scripts/functions/unary/scalar/StopTestLoops_fn.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTestLoops_fn.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
A = read($1, rows=$2, cols=1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTestLoops_for.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTestLoops_for.dml b/src/test/scripts/functions/unary/scalar/StopTestLoops_for.dml
index 381c3c7..2ad3a89 100644
--- a/src/test/scripts/functions/unary/scalar/StopTestLoops_for.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTestLoops_for.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
A = read($1, rows=$2, cols=1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTestLoops_parfor.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTestLoops_parfor.dml b/src/test/scripts/functions/unary/scalar/StopTestLoops_parfor.dml
index bd0bfe9..1675071 100644
--- a/src/test/scripts/functions/unary/scalar/StopTestLoops_parfor.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTestLoops_parfor.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
A = rand(rows=10, cols=1, seed=10);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/StopTestLoops_while.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/StopTestLoops_while.dml b/src/test/scripts/functions/unary/scalar/StopTestLoops_while.dml
index 48b7657..aee45e5 100644
--- a/src/test/scripts/functions/unary/scalar/StopTestLoops_while.dml
+++ b/src/test/scripts/functions/unary/scalar/StopTestLoops_while.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.StopTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.StopTest.java
A = read($1, rows=$2, cols=1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/unary/scalar/TanTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/scalar/TanTest.dml b/src/test/scripts/functions/unary/scalar/TanTest.dml
index 2d4e030..0ee988a 100644
--- a/src/test/scripts/functions/unary/scalar/TanTest.dml
+++ b/src/test/scripts/functions/unary/scalar/TanTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.unary.scalar.TanTest.java
+# junit test class: org.apache.sysml.test.integration.functions.unary.scalar.TanTest.java
$$readhelper$$
[78/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
Update source references based on new package structure
The refactor from com.ibm.bi.dml to org.apache.sysml
is being done in two phases to avoid loosing the file
history. In this phase, all references to old package
is being updated to reference the new project structure.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7cd7dc2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7cd7dc2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7cd7dc2b
Branch: refs/heads/master
Commit: 7cd7dc2be83ea73c700b2bebe50e4f37bd275974
Parents: 276d925
Author: Luciano Resende <lr...@apache.org>
Authored: Tue Dec 1 13:58:16 2015 -0800
Committer: Luciano Resende <lr...@apache.org>
Committed: Thu Dec 3 10:44:00 2015 -0800
----------------------------------------------------------------------
.gitignore | 16 +-
bin/systemml | 4 +-
bin/systemml.bat | 2 +-
.../SystemML_Language_Reference.html | 4 +-
docs/dml-language-reference.md | 4 +-
docs/mlcontext-programming-guide.md | 96 ++---
docs/quick-start-guide.md | 2 +-
pom.xml | 4 +-
scripts/algorithms/decision-tree.dml | 2 +-
scripts/algorithms/random-forest.dml | 2 +-
scripts/staging/Lanczos.dml | 2 +-
.../java/org/apache/sysml/api/DMLException.java | 2 +-
.../java/org/apache/sysml/api/DMLScript.java | 86 ++---
src/main/java/org/apache/sysml/api/MLBlock.java | 6 +-
.../java/org/apache/sysml/api/MLContext.java | 98 ++---
.../org/apache/sysml/api/MLContextProxy.java | 12 +-
.../java/org/apache/sysml/api/MLMatrix.java | 20 +-
.../java/org/apache/sysml/api/MLOutput.java | 16 +-
.../org/apache/sysml/api/jmlc/Connection.java | 42 +--
.../org/apache/sysml/api/jmlc/JMLCUtils.java | 18 +-
.../apache/sysml/api/jmlc/PreparedScript.java | 42 +--
.../apache/sysml/api/jmlc/ResultVariables.java | 12 +-
.../apache/sysml/api/ml/LogisticRegression.java | 28 +-
.../sysml/api/ml/LogisticRegressionModel.java | 20 +-
.../functions/ConvertSingleColumnToString.java | 2 +-
.../api/ml/functions/ConvertVectorToDouble.java | 2 +-
.../api/monitoring/InstructionComparator.java | 2 +-
.../apache/sysml/api/monitoring/Location.java | 2 +-
.../api/monitoring/SparkMonitoringUtil.java | 12 +-
.../org/apache/sysml/api/python/SystemML.py | 2 +-
.../apache/sysml/conf/ConfigurationManager.java | 2 +-
.../java/org/apache/sysml/conf/DMLConfig.java | 8 +-
.../sysml/debug/DMLBreakpointManager.java | 6 +-
.../org/apache/sysml/debug/DMLDebugger.java | 10 +-
.../sysml/debug/DMLDebuggerException.java | 4 +-
.../sysml/debug/DMLDebuggerFunctions.java | 38 +-
.../sysml/debug/DMLDebuggerInterface.java | 2 +-
.../sysml/debug/DMLDebuggerProgramInfo.java | 34 +-
.../org/apache/sysml/debug/DMLDisassemble.java | 24 +-
.../java/org/apache/sysml/debug/DMLFrame.java | 4 +-
.../apache/sysml/debug/DMLProgramCounter.java | 4 +-
.../java/org/apache/sysml/debug/DebugState.java | 8 +-
.../java/org/apache/sysml/hops/AggBinaryOp.java | 68 ++--
.../java/org/apache/sysml/hops/AggUnaryOp.java | 40 +-
.../java/org/apache/sysml/hops/BinaryOp.java | 68 ++--
.../java/org/apache/sysml/hops/DataGenOp.java | 34 +-
src/main/java/org/apache/sysml/hops/DataOp.java | 18 +-
.../java/org/apache/sysml/hops/FunctionOp.java | 18 +-
src/main/java/org/apache/sysml/hops/Hop.java | 374 +++++++++----------
.../org/apache/sysml/hops/HopsException.java | 4 +-
.../java/org/apache/sysml/hops/IndexingOp.java | 26 +-
.../org/apache/sysml/hops/LeftIndexingOp.java | 30 +-
.../java/org/apache/sysml/hops/LiteralOp.java | 16 +-
.../java/org/apache/sysml/hops/MemoTable.java | 12 +-
.../org/apache/sysml/hops/OptimizerUtils.java | 44 +--
.../sysml/hops/ParameterizedBuiltinOp.java | 34 +-
.../org/apache/sysml/hops/QuaternaryOp.java | 68 ++--
.../java/org/apache/sysml/hops/ReorgOp.java | 24 +-
.../java/org/apache/sysml/hops/TernaryOp.java | 42 +--
.../java/org/apache/sysml/hops/UnaryOp.java | 42 +--
.../sysml/hops/cost/CostEstimationWrapper.java | 24 +-
.../apache/sysml/hops/cost/CostEstimator.java | 84 ++---
.../sysml/hops/cost/CostEstimatorNumMRJobs.java | 8 +-
.../hops/cost/CostEstimatorStaticRuntime.java | 86 ++---
.../org/apache/sysml/hops/cost/VarStats.java | 2 +-
.../sysml/hops/globalopt/GDFEnumOptimizer.java | 58 +--
.../sysml/hops/globalopt/GlobalOptimizer.java | 12 +-
.../hops/globalopt/GlobalOptimizerWrapper.java | 24 +-
.../hops/globalopt/InterestingProperties.java | 2 +-
.../sysml/hops/globalopt/MemoStructure.java | 4 +-
.../org/apache/sysml/hops/globalopt/Plan.java | 20 +-
.../apache/sysml/hops/globalopt/PlanSet.java | 8 +-
.../sysml/hops/globalopt/RewriteConfig.java | 8 +-
.../apache/sysml/hops/globalopt/Summary.java | 2 +-
.../globalopt/gdfgraph/GDFCrossBlockNode.java | 6 +-
.../sysml/hops/globalopt/gdfgraph/GDFGraph.java | 4 +-
.../hops/globalopt/gdfgraph/GDFLoopNode.java | 4 +-
.../sysml/hops/globalopt/gdfgraph/GDFNode.java | 30 +-
.../hops/globalopt/gdfgraph/GraphBuilder.java | 38 +-
.../gdfresolve/GDFMismatchHeuristic.java | 4 +-
.../GDFMismatchHeuristicBlocksizeOrFirst.java | 4 +-
.../gdfresolve/GDFMismatchHeuristicFirst.java | 4 +-
.../gdfresolve/MismatchHeuristicFactory.java | 6 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 102 ++---
.../hops/recompile/LiteralReplacement.java | 46 +--
.../sysml/hops/recompile/RecompileStatus.java | 6 +-
.../apache/sysml/hops/recompile/Recompiler.java | 142 +++----
.../sysml/hops/rewrite/HopDagValidator.java | 14 +-
.../sysml/hops/rewrite/HopRewriteRule.java | 6 +-
.../sysml/hops/rewrite/HopRewriteUtils.java | 72 ++--
.../hops/rewrite/ProgramRewriteStatus.java | 2 +-
.../sysml/hops/rewrite/ProgramRewriter.java | 34 +-
.../RewriteAlgebraicSimplificationDynamic.java | 50 +--
.../RewriteAlgebraicSimplificationStatic.java | 48 +--
.../rewrite/RewriteBlockSizeAndReblock.java | 26 +-
.../RewriteCommonSubexpressionElimination.java | 10 +-
.../hops/rewrite/RewriteConstantFolding.java | 48 +--
.../rewrite/RewriteForLoopVectorization.java | 42 +--
.../rewrite/RewriteIndexingVectorization.java | 16 +-
.../RewriteInjectSparkLoopCheckpointing.java | 26 +-
.../RewriteInjectSparkPReadCheckpointing.java | 12 +-
.../RewriteMatrixMultChainOptimization.java | 12 +-
.../RewriteRemovePersistentReadWrite.java | 12 +-
.../rewrite/RewriteRemoveReadAfterWrite.java | 14 +-
.../RewriteRemoveUnnecessaryBranches.java | 14 +-
.../rewrite/RewriteRemoveUnnecessaryCasts.java | 12 +-
.../RewriteSplitDagDataDependentOperators.java | 46 +--
.../rewrite/RewriteSplitDagUnknownCSVRead.java | 20 +-
.../RewriteTransientWriteParentHandling.java | 10 +-
.../hops/rewrite/StatementBlockRewriteRule.java | 6 +-
.../java/org/apache/sysml/lops/Aggregate.java | 12 +-
.../java/org/apache/sysml/lops/AppendCP.java | 10 +-
.../java/org/apache/sysml/lops/AppendG.java | 10 +-
.../org/apache/sysml/lops/AppendGAlignedSP.java | 12 +-
.../java/org/apache/sysml/lops/AppendM.java | 10 +-
.../java/org/apache/sysml/lops/AppendR.java | 10 +-
src/main/java/org/apache/sysml/lops/Binary.java | 10 +-
.../java/org/apache/sysml/lops/BinaryM.java | 12 +-
.../org/apache/sysml/lops/BinaryScalar.java | 10 +-
.../org/apache/sysml/lops/BinaryUAggChain.java | 12 +-
.../java/org/apache/sysml/lops/CSVReBlock.java | 18 +-
.../org/apache/sysml/lops/CentralMoment.java | 10 +-
.../java/org/apache/sysml/lops/Checkpoint.java | 12 +-
.../java/org/apache/sysml/lops/CoVariance.java | 12 +-
.../org/apache/sysml/lops/CombineBinary.java | 12 +-
.../org/apache/sysml/lops/CombineTernary.java | 12 +-
.../org/apache/sysml/lops/CombineUnary.java | 10 +-
.../sysml/lops/CumulativeOffsetBinary.java | 12 +-
.../sysml/lops/CumulativePartialAggregate.java | 12 +-
.../sysml/lops/CumulativeSplitAggregate.java | 10 +-
src/main/java/org/apache/sysml/lops/Data.java | 20 +-
.../java/org/apache/sysml/lops/DataGen.java | 20 +-
.../org/apache/sysml/lops/DataPartition.java | 14 +-
.../org/apache/sysml/lops/FunctionCallCP.java | 18 +-
src/main/java/org/apache/sysml/lops/Group.java | 10 +-
.../org/apache/sysml/lops/GroupedAggregate.java | 12 +-
.../java/org/apache/sysml/lops/LeftIndex.java | 12 +-
src/main/java/org/apache/sysml/lops/Lop.java | 12 +-
.../org/apache/sysml/lops/LopProperties.java | 6 +-
.../org/apache/sysml/lops/LopsException.java | 4 +-
src/main/java/org/apache/sysml/lops/MMCJ.java | 12 +-
src/main/java/org/apache/sysml/lops/MMRJ.java | 10 +-
src/main/java/org/apache/sysml/lops/MMTSJ.java | 10 +-
src/main/java/org/apache/sysml/lops/MMZip.java | 12 +-
.../java/org/apache/sysml/lops/MapMult.java | 14 +-
.../org/apache/sysml/lops/MapMultChain.java | 12 +-
.../org/apache/sysml/lops/OutputParameters.java | 4 +-
src/main/java/org/apache/sysml/lops/PMMJ.java | 12 +-
.../java/org/apache/sysml/lops/PMapMult.java | 12 +-
.../apache/sysml/lops/ParameterizedBuiltin.java | 16 +-
.../org/apache/sysml/lops/PartialAggregate.java | 14 +-
.../java/org/apache/sysml/lops/PickByCount.java | 12 +-
.../apache/sysml/lops/RangeBasedReIndex.java | 14 +-
.../java/org/apache/sysml/lops/ReBlock.java | 14 +-
src/main/java/org/apache/sysml/lops/RepMat.java | 12 +-
.../java/org/apache/sysml/lops/SortKeys.java | 12 +-
.../java/org/apache/sysml/lops/Ternary.java | 10 +-
.../org/apache/sysml/lops/TernaryAggregate.java | 10 +-
.../java/org/apache/sysml/lops/Transform.java | 10 +-
.../org/apache/sysml/lops/UAggOuterChain.java | 12 +-
src/main/java/org/apache/sysml/lops/Unary.java | 12 +-
.../java/org/apache/sysml/lops/UnaryCP.java | 10 +-
.../apache/sysml/lops/WeightedCrossEntropy.java | 12 +-
.../sysml/lops/WeightedCrossEntropyR.java | 14 +-
.../org/apache/sysml/lops/WeightedDivMM.java | 14 +-
.../org/apache/sysml/lops/WeightedDivMMR.java | 14 +-
.../org/apache/sysml/lops/WeightedSigmoid.java | 12 +-
.../org/apache/sysml/lops/WeightedSigmoidR.java | 14 +-
.../apache/sysml/lops/WeightedSquaredLoss.java | 12 +-
.../apache/sysml/lops/WeightedSquaredLossR.java | 14 +-
.../org/apache/sysml/lops/WeightedUnaryMM.java | 14 +-
.../org/apache/sysml/lops/WeightedUnaryMMR.java | 16 +-
.../java/org/apache/sysml/lops/ZeroOut.java | 12 +-
.../java/org/apache/sysml/lops/compile/Dag.java | 124 +++---
.../org/apache/sysml/lops/compile/JobType.java | 12 +-
.../sysml/lops/compile/LopComparator.java | 4 +-
.../apache/sysml/lops/runtime/RunMRJobs.java | 100 ++---
.../org/apache/sysml/parser/AParserWrapper.java | 6 +-
.../sysml/parser/AssignmentStatement.java | 6 +-
.../apache/sysml/parser/BinaryExpression.java | 2 +-
.../apache/sysml/parser/BooleanExpression.java | 2 +-
.../apache/sysml/parser/BooleanIdentifier.java | 2 +-
.../sysml/parser/BuiltinFunctionExpression.java | 4 +-
.../sysml/parser/ConditionalPredicate.java | 2 +-
.../apache/sysml/parser/ConstIdentifier.java | 2 +-
.../apache/sysml/parser/DMLParseException.java | 2 +-
.../org/apache/sysml/parser/DMLProgram.java | 44 +--
.../org/apache/sysml/parser/DMLTranslator.java | 80 ++--
.../org/apache/sysml/parser/DataExpression.java | 14 +-
.../org/apache/sysml/parser/DataIdentifier.java | 2 +-
.../apache/sysml/parser/DoubleIdentifier.java | 4 +-
.../org/apache/sysml/parser/Expression.java | 4 +-
.../sysml/parser/ExternalFunctionStatement.java | 2 +-
.../org/apache/sysml/parser/ForStatement.java | 2 +-
.../apache/sysml/parser/ForStatementBlock.java | 16 +-
.../sysml/parser/FunctionCallIdentifier.java | 2 +-
.../apache/sysml/parser/FunctionStatement.java | 4 +-
.../sysml/parser/FunctionStatementBlock.java | 12 +-
.../org/apache/sysml/parser/Identifier.java | 4 +-
.../org/apache/sysml/parser/IfStatement.java | 2 +-
.../apache/sysml/parser/IfStatementBlock.java | 14 +-
.../apache/sysml/parser/ImportStatement.java | 2 +-
.../apache/sysml/parser/IndexedIdentifier.java | 4 +-
.../org/apache/sysml/parser/IntIdentifier.java | 2 +-
.../apache/sysml/parser/IterablePredicate.java | 4 +-
.../apache/sysml/parser/LanguageException.java | 4 +-
.../sysml/parser/LiveVariableAnalysis.java | 2 +-
.../sysml/parser/MultiAssignmentStatement.java | 6 +-
.../apache/sysml/parser/OutputStatement.java | 8 +-
.../apache/sysml/parser/ParForStatement.java | 2 +-
.../sysml/parser/ParForStatementBlock.java | 34 +-
.../sysml/parser/ParameterExpression.java | 2 +-
.../ParameterizedBuiltinFunctionExpression.java | 6 +-
.../org/apache/sysml/parser/ParseException.java | 2 +-
.../org/apache/sysml/parser/PathStatement.java | 2 +-
.../org/apache/sysml/parser/PrintStatement.java | 6 +-
.../sysml/parser/RelationalExpression.java | 4 +-
.../java/org/apache/sysml/parser/Statement.java | 2 +-
.../org/apache/sysml/parser/StatementBlock.java | 28 +-
.../apache/sysml/parser/StringIdentifier.java | 2 +-
.../java/org/apache/sysml/parser/Token.java | 2 +-
.../org/apache/sysml/parser/VariableSet.java | 2 +-
.../org/apache/sysml/parser/WhileStatement.java | 2 +-
.../sysml/parser/WhileStatementBlock.java | 12 +-
.../sysml/parser/antlr4/DMLParserWrapper.java | 50 +--
.../java/org/apache/sysml/parser/antlr4/Dml.g4 | 6 +-
.../antlr4/DmlSyntacticErrorListener.java | 4 +-
.../parser/antlr4/DmlSyntacticValidator.java | 174 ++++-----
.../antlr4/DmlSyntacticValidatorHelper.java | 48 +--
.../sysml/parser/antlr4/ExpressionInfo.java | 10 +-
.../sysml/parser/antlr4/StatementInfo.java | 6 +-
.../sysml/parser/python/ExpressionInfo.java | 4 +-
.../sysml/parser/python/PyDMLParserWrapper.java | 48 +--
.../org/apache/sysml/parser/python/Pydml.g4 | 10 +-
.../python/PydmlSyntacticErrorListener.java | 4 +-
.../parser/python/PydmlSyntacticValidator.java | 182 ++++-----
.../python/PydmlSyntacticValidatorHelper.java | 48 +--
.../sysml/parser/python/StatementInfo.java | 4 +-
.../sysml/runtime/DMLRuntimeException.java | 4 +-
.../sysml/runtime/DMLScriptException.java | 2 +-
.../DMLUnsupportedOperationException.java | 4 +-
.../ExternalFunctionProgramBlock.java | 80 ++--
.../ExternalFunctionProgramBlockCP.java | 34 +-
.../runtime/controlprogram/ForProgramBlock.java | 30 +-
.../controlprogram/FunctionProgramBlock.java | 22 +-
.../runtime/controlprogram/IfProgramBlock.java | 40 +-
.../controlprogram/LocalVariableMap.java | 16 +-
.../controlprogram/ParForProgramBlock.java | 146 ++++----
.../sysml/runtime/controlprogram/Program.java | 12 +-
.../runtime/controlprogram/ProgramBlock.java | 50 +--
.../controlprogram/WhileProgramBlock.java | 40 +-
.../controlprogram/caching/ByteBuffer.java | 6 +-
.../controlprogram/caching/CacheDataInput.java | 6 +-
.../controlprogram/caching/CacheDataOutput.java | 6 +-
.../controlprogram/caching/CacheException.java | 4 +-
.../caching/CacheIOException.java | 2 +-
.../controlprogram/caching/CacheStatistics.java | 2 +-
.../caching/CacheStatusException.java | 2 +-
.../controlprogram/caching/CacheableData.java | 20 +-
.../controlprogram/caching/LazyWriteBuffer.java | 10 +-
.../controlprogram/caching/MatrixObject.java | 52 +--
.../controlprogram/caching/PageCache.java | 2 +-
.../context/ExecutionContext.java | 52 +--
.../context/ExecutionContextFactory.java | 6 +-
.../context/SparkExecutionContext.java | 66 ++--
.../controlprogram/parfor/DataPartitioner.java | 26 +-
.../parfor/DataPartitionerLocal.java | 38 +-
.../parfor/DataPartitionerRemoteMR.java | 30 +-
.../parfor/DataPartitionerRemoteMapper.java | 28 +-
.../parfor/DataPartitionerRemoteReducer.java | 16 +-
.../parfor/DataPartitionerRemoteSpark.java | 28 +-
.../DataPartitionerRemoteSparkMapper.java | 22 +-
.../DataPartitionerRemoteSparkReducer.java | 16 +-
.../controlprogram/parfor/LocalParWorker.java | 12 +-
.../controlprogram/parfor/LocalTaskQueue.java | 2 +-
.../controlprogram/parfor/ParForBody.java | 8 +-
.../controlprogram/parfor/ParWorker.java | 24 +-
.../controlprogram/parfor/ProgramConverter.java | 110 +++---
.../controlprogram/parfor/RemoteDPParForMR.java | 46 +--
.../parfor/RemoteDPParForSpark.java | 32 +-
.../parfor/RemoteDPParForSparkWorker.java | 30 +-
.../parfor/RemoteDPParWorkerReducer.java | 40 +-
.../parfor/RemoteParForColocatedFileSplit.java | 8 +-
.../RemoteParForColocatedNLineInputFormat.java | 6 +-
.../parfor/RemoteParForJobReturn.java | 4 +-
.../controlprogram/parfor/RemoteParForMR.java | 38 +-
.../parfor/RemoteParForSpark.java | 16 +-
.../parfor/RemoteParForSparkWorker.java | 12 +-
.../parfor/RemoteParForUtils.java | 30 +-
.../parfor/RemoteParWorkerMapper.java | 30 +-
.../controlprogram/parfor/ResultMerge.java | 10 +-
.../parfor/ResultMergeLocalAutomatic.java | 14 +-
.../parfor/ResultMergeLocalFile.java | 48 +--
.../parfor/ResultMergeLocalMemory.java | 24 +-
.../parfor/ResultMergeRemoteGrouping.java | 2 +-
.../parfor/ResultMergeRemoteMR.java | 42 +--
.../parfor/ResultMergeRemoteMapper.java | 20 +-
.../parfor/ResultMergeRemotePartitioning.java | 8 +-
.../parfor/ResultMergeRemoteReducer.java | 22 +-
.../parfor/ResultMergeRemoteSorting.java | 2 +-
.../parfor/ResultMergeRemoteSpark.java | 34 +-
.../parfor/ResultMergeRemoteSparkWCompare.java | 12 +-
.../parfor/ResultMergeTaggedMatrixIndexes.java | 4 +-
.../runtime/controlprogram/parfor/Task.java | 4 +-
.../controlprogram/parfor/TaskPartitioner.java | 6 +-
.../parfor/TaskPartitionerFactoring.java | 10 +-
.../parfor/TaskPartitionerFactoringCmax.java | 4 +-
.../parfor/TaskPartitionerFactoringCmin.java | 4 +-
.../parfor/TaskPartitionerFixedsize.java | 10 +-
.../parfor/TaskPartitionerNaive.java | 4 +-
.../parfor/TaskPartitionerStatic.java | 4 +-
.../parfor/mqo/MergedMRJobInstruction.java | 8 +-
.../parfor/mqo/PiggybackingWorker.java | 10 +-
.../mqo/PiggybackingWorkerTimeSequential.java | 12 +-
.../PiggybackingWorkerUtilDecayParallel.java | 14 +-
.../mqo/PiggybackingWorkerUtilTimeParallel.java | 12 +-
.../parfor/mqo/RuntimePiggybacking.java | 14 +-
.../parfor/mqo/RuntimePiggybackingUtils.java | 6 +-
.../parfor/opt/CostEstimator.java | 10 +-
.../parfor/opt/CostEstimatorHops.java | 20 +-
.../parfor/opt/CostEstimatorRuntime.java | 14 +-
.../controlprogram/parfor/opt/CostFunction.java | 2 +-
.../controlprogram/parfor/opt/MemoTable.java | 4 +-
.../parfor/opt/MemoTableEntry.java | 4 +-
.../controlprogram/parfor/opt/OptNode.java | 10 +-
.../parfor/opt/OptNodeStatistics.java | 4 +-
.../controlprogram/parfor/opt/OptTree.java | 4 +-
.../parfor/opt/OptTreeConverter.java | 104 +++---
.../parfor/opt/OptTreePlanChecker.java | 48 +--
.../parfor/opt/OptTreePlanMapping.java | 4 +-
.../parfor/opt/OptTreePlanMappingAbstract.java | 12 +-
.../parfor/opt/OptTreePlanMappingRuntime.java | 6 +-
.../parfor/opt/OptimizationWrapper.java | 86 ++---
.../controlprogram/parfor/opt/Optimizer.java | 20 +-
.../parfor/opt/OptimizerConstrained.java | 38 +-
.../parfor/opt/OptimizerHeuristic.java | 8 +-
.../parfor/opt/OptimizerRuleBased.java | 128 +++----
.../controlprogram/parfor/opt/PerfTestDef.java | 10 +-
.../parfor/opt/PerfTestExtFunctCP.java | 20 +-
.../parfor/opt/PerfTestMemoryObserver.java | 2 +-
.../controlprogram/parfor/opt/PerfTestTool.java | 82 ++--
.../parfor/opt/PerfTestToolRegression.dml | 6 +-
.../parfor/opt/ProgramRecompiler.java | 78 ++--
.../parfor/stat/InfrastructureAnalyzer.java | 6 +-
.../controlprogram/parfor/stat/Stat.java | 2 +-
.../parfor/stat/StatisticMonitor.java | 10 +-
.../controlprogram/parfor/stat/Timing.java | 2 +-
.../controlprogram/parfor/util/Cell.java | 2 +-
.../controlprogram/parfor/util/IDHandler.java | 2 +-
.../controlprogram/parfor/util/IDSequence.java | 2 +-
.../parfor/util/PairWritableBlock.java | 6 +-
.../parfor/util/PairWritableCell.java | 6 +-
.../parfor/util/StagingFileUtils.java | 8 +-
.../sysml/runtime/functionobjects/And.java | 2 +-
.../sysml/runtime/functionobjects/Builtin.java | 10 +-
.../sysml/runtime/functionobjects/CM.java | 12 +-
.../sysml/runtime/functionobjects/COV.java | 10 +-
.../sysml/runtime/functionobjects/CTable.java | 14 +-
.../runtime/functionobjects/DiagIndex.java | 10 +-
.../sysml/runtime/functionobjects/Divide.java | 2 +-
.../sysml/runtime/functionobjects/Equals.java | 2 +-
.../runtime/functionobjects/FileFunction.java | 2 +-
.../runtime/functionobjects/FunctionObject.java | 12 +-
.../runtime/functionobjects/GreaterThan.java | 2 +-
.../functionobjects/GreaterThanEquals.java | 2 +-
.../runtime/functionobjects/IndexFunction.java | 2 +-
.../runtime/functionobjects/IntegerDivide.java | 4 +-
.../runtime/functionobjects/KahanFunction.java | 4 +-
.../runtime/functionobjects/KahanPlus.java | 8 +-
.../runtime/functionobjects/KahanPlusSq.java | 8 +-
.../sysml/runtime/functionobjects/LessThan.java | 2 +-
.../runtime/functionobjects/LessThanEquals.java | 2 +-
.../sysml/runtime/functionobjects/MaxIndex.java | 10 +-
.../sysml/runtime/functionobjects/Mean.java | 8 +-
.../sysml/runtime/functionobjects/MinIndex.java | 10 +-
.../sysml/runtime/functionobjects/Minus.java | 2 +-
.../runtime/functionobjects/Minus1Multiply.java | 2 +-
.../sysml/runtime/functionobjects/MinusNz.java | 2 +-
.../sysml/runtime/functionobjects/Modulus.java | 2 +-
.../sysml/runtime/functionobjects/Multiply.java | 2 +-
.../runtime/functionobjects/Multiply2.java | 2 +-
.../sysml/runtime/functionobjects/Not.java | 2 +-
.../runtime/functionobjects/NotEquals.java | 2 +-
.../functionobjects/OffsetColumnIndex.java | 10 +-
.../sysml/runtime/functionobjects/Or.java | 2 +-
.../functionobjects/ParameterizedBuiltin.java | 6 +-
.../sysml/runtime/functionobjects/Plus.java | 4 +-
.../sysml/runtime/functionobjects/Power.java | 2 +-
.../sysml/runtime/functionobjects/Power2.java | 2 +-
.../runtime/functionobjects/ReduceAll.java | 10 +-
.../runtime/functionobjects/ReduceCol.java | 10 +-
.../runtime/functionobjects/ReduceDiag.java | 10 +-
.../runtime/functionobjects/ReduceRow.java | 10 +-
.../runtime/functionobjects/RemoveFile.java | 6 +-
.../runtime/functionobjects/RenameFile.java | 6 +-
.../runtime/functionobjects/SortIndex.java | 6 +-
.../runtime/functionobjects/SwapIndex.java | 10 +-
.../runtime/functionobjects/ValueFunction.java | 2 +-
.../instructions/CPInstructionParser.java | 82 ++--
.../sysml/runtime/instructions/Instruction.java | 14 +-
.../runtime/instructions/InstructionParser.java | 14 +-
.../runtime/instructions/InstructionUtils.java | 112 +++---
.../instructions/MRInstructionParser.java | 116 +++---
.../runtime/instructions/MRJobInstruction.java | 52 +--
.../instructions/SPInstructionParser.java | 108 +++---
.../cp/AggregateBinaryCPInstruction.java | 26 +-
.../cp/AggregateTernaryCPInstruction.java | 22 +-
.../cp/AggregateUnaryCPInstruction.java | 34 +-
.../instructions/cp/AppendCPInstruction.java | 22 +-
.../cp/ArithmeticBinaryCPInstruction.java | 16 +-
.../instructions/cp/BinaryCPInstruction.java | 8 +-
.../cp/BooleanBinaryCPInstruction.java | 18 +-
.../runtime/instructions/cp/BooleanObject.java | 4 +-
.../cp/BooleanUnaryCPInstruction.java | 16 +-
.../instructions/cp/BreakPointInstruction.java | 12 +-
.../cp/BuiltinBinaryCPInstruction.java | 22 +-
.../cp/BuiltinUnaryCPInstruction.java | 24 +-
.../runtime/instructions/cp/CM_COV_Object.java | 14 +-
.../runtime/instructions/cp/CPInstruction.java | 18 +-
.../runtime/instructions/cp/CPOperand.java | 8 +-
.../cp/CentralMomentCPInstruction.java | 24 +-
.../cp/ComputationCPInstruction.java | 8 +-
.../cp/CovarianceCPInstruction.java | 22 +-
.../sysml/runtime/instructions/cp/Data.java | 12 +-
.../instructions/cp/DataGenCPInstruction.java | 28 +-
.../cp/DataPartitionCPInstruction.java | 34 +-
.../runtime/instructions/cp/DoubleObject.java | 6 +-
.../instructions/cp/FileCPInstruction.java | 22 +-
.../runtime/instructions/cp/FileObject.java | 6 +-
.../cp/FunctionCallCPInstruction.java | 32 +-
.../runtime/instructions/cp/IntObject.java | 4 +-
.../runtime/instructions/cp/KahanObject.java | 6 +-
.../instructions/cp/MMChainCPInstruction.java | 18 +-
.../instructions/cp/MMTSJCPInstruction.java | 18 +-
.../cp/MatrixBuiltinCPInstruction.java | 18 +-
.../cp/MatrixIndexingCPInstruction.java | 26 +-
.../cp/MatrixMatrixArithmeticCPInstruction.java | 14 +-
.../cp/MatrixMatrixBuiltinCPInstruction.java | 18 +-
.../cp/MatrixMatrixRelationalCPInstruction.java | 14 +-
.../cp/MatrixReshapeCPInstruction.java | 20 +-
.../cp/MatrixScalarBuiltinCPInstruction.java | 16 +-
.../cp/MultiReturnBuiltinCPInstruction.java | 24 +-
.../instructions/cp/PMMJCPInstruction.java | 16 +-
.../cp/ParameterizedBuiltinCPInstruction.java | 34 +-
.../cp/QuantilePickCPInstruction.java | 30 +-
.../cp/QuantileSortCPInstruction.java | 24 +-
.../cp/QuaternaryCPInstruction.java | 28 +-
.../cp/RelationalBinaryCPInstruction.java | 14 +-
.../instructions/cp/ReorgCPInstruction.java | 28 +-
.../cp/ScalarBuiltinCPInstruction.java | 16 +-
.../cp/ScalarMatrixArithmeticCPInstruction.java | 16 +-
.../cp/ScalarMatrixRelationalCPInstruction.java | 16 +-
.../runtime/instructions/cp/ScalarObject.java | 6 +-
.../cp/ScalarScalarArithmeticCPInstruction.java | 16 +-
.../cp/ScalarScalarBuiltinCPInstruction.java | 14 +-
.../cp/ScalarScalarRelationalCPInstruction.java | 10 +-
.../cp/StringInitCPInstruction.java | 18 +-
.../runtime/instructions/cp/StringObject.java | 8 +-
.../instructions/cp/TernaryCPInstruction.java | 28 +-
.../cp/UaggOuterChainCPInstruction.java | 30 +-
.../instructions/cp/UnaryCPInstruction.java | 12 +-
.../instructions/cp/VariableCPInstruction.java | 50 +--
.../cpfile/MatrixIndexingCPFileInstruction.java | 34 +-
.../ParameterizedBuiltinCPFileInstruction.java | 66 ++--
.../mr/AggregateBinaryInstruction.java | 40 +-
.../instructions/mr/AggregateInstruction.java | 20 +-
.../mr/AggregateUnaryInstruction.java | 28 +-
.../instructions/mr/AppendGInstruction.java | 22 +-
.../instructions/mr/AppendInstruction.java | 18 +-
.../instructions/mr/AppendMInstruction.java | 28 +-
.../instructions/mr/AppendRInstruction.java | 20 +-
.../mr/BinUaggChainInstruction.java | 26 +-
.../instructions/mr/BinaryInstruction.java | 24 +-
.../instructions/mr/BinaryMInstruction.java | 30 +-
.../mr/BinaryMRInstructionBase.java | 4 +-
.../instructions/mr/CM_N_COVInstruction.java | 28 +-
.../instructions/mr/CSVReblockInstruction.java | 8 +-
.../instructions/mr/CSVWriteInstruction.java | 18 +-
.../mr/CombineBinaryInstruction.java | 18 +-
.../mr/CombineTernaryInstruction.java | 18 +-
.../mr/CombineUnaryInstruction.java | 18 +-
.../mr/CumulativeAggregateInstruction.java | 28 +-
.../mr/CumulativeOffsetInstruction.java | 28 +-
.../mr/CumulativeSplitInstruction.java | 22 +-
.../instructions/mr/DataGenMRInstruction.java | 6 +-
.../mr/DataPartitionMRInstruction.java | 10 +-
.../mr/GroupedAggregateInstruction.java | 30 +-
.../mr/IDistributedCacheConsumer.java | 2 +-
.../instructions/mr/MMTSJMRInstruction.java | 22 +-
.../runtime/instructions/mr/MRInstruction.java | 18 +-
.../mr/MapMultChainInstruction.java | 26 +-
.../mr/MatrixReshapeMRInstruction.java | 24 +-
.../instructions/mr/PMMJMRInstruction.java | 30 +-
.../mr/ParameterizedBuiltinMRInstruction.java | 26 +-
.../instructions/mr/PickByCountInstruction.java | 20 +-
.../instructions/mr/QuaternaryInstruction.java | 54 +--
.../instructions/mr/RandInstruction.java | 20 +-
.../mr/RangeBasedReIndexInstruction.java | 26 +-
.../instructions/mr/ReblockInstruction.java | 16 +-
.../mr/RemoveEmptyMRInstruction.java | 22 +-
.../instructions/mr/ReorgInstruction.java | 28 +-
.../instructions/mr/ReplicateInstruction.java | 20 +-
.../instructions/mr/ScalarInstruction.java | 24 +-
.../runtime/instructions/mr/SeqInstruction.java | 20 +-
.../instructions/mr/TernaryInstruction.java | 26 +-
.../mr/UaggOuterChainInstruction.java | 44 +--
.../instructions/mr/UnaryInstruction.java | 24 +-
.../instructions/mr/UnaryMRInstructionBase.java | 4 +-
.../instructions/mr/ZeroOutInstruction.java | 26 +-
.../spark/AggregateTernarySPInstruction.java | 34 +-
.../spark/AggregateUnarySPInstruction.java | 44 +--
.../spark/AppendGAlignedSPInstruction.java | 28 +-
.../spark/AppendGSPInstruction.java | 30 +-
.../spark/AppendMSPInstruction.java | 38 +-
.../spark/AppendRSPInstruction.java | 26 +-
.../spark/ArithmeticBinarySPInstruction.java | 22 +-
.../spark/BinUaggChainSPInstruction.java | 24 +-
.../instructions/spark/BinarySPInstruction.java | 44 +--
.../spark/BuiltinBinarySPInstruction.java | 30 +-
.../spark/BuiltinUnarySPInstruction.java | 22 +-
.../spark/CSVReblockSPInstruction.java | 34 +-
.../spark/CentralMomentSPInstruction.java | 36 +-
.../spark/CheckpointSPInstruction.java | 36 +-
.../spark/ComputationSPInstruction.java | 12 +-
.../spark/CovarianceSPInstruction.java | 32 +-
.../instructions/spark/CpmmSPInstruction.java | 34 +-
.../spark/CumulativeAggregateSPInstruction.java | 26 +-
.../spark/CumulativeOffsetSPInstruction.java | 34 +-
.../spark/MapmmChainSPInstruction.java | 28 +-
.../instructions/spark/MapmmSPInstruction.java | 46 +--
.../MatrixBVectorArithmeticSPInstruction.java | 14 +-
.../MatrixBVectorBuiltinSPInstruction.java | 14 +-
.../MatrixBVectorRelationalSPInstruction.java | 14 +-
.../spark/MatrixBuiltinSPInstruction.java | 20 +-
.../spark/MatrixIndexingSPInstruction.java | 46 +--
.../MatrixMatrixArithmeticSPInstruction.java | 12 +-
.../spark/MatrixMatrixBuiltinSPInstruction.java | 12 +-
.../MatrixMatrixRelationalSPInstruction.java | 12 +-
.../spark/MatrixReshapeSPInstruction.java | 34 +-
.../MatrixScalarArithmeticSPInstruction.java | 12 +-
.../spark/MatrixScalarBuiltinSPInstruction.java | 12 +-
.../MatrixScalarRelationalSPInstruction.java | 12 +-
.../instructions/spark/PMapmmSPInstruction.java | 40 +-
.../ParameterizedBuiltinSPInstruction.java | 76 ++--
.../instructions/spark/PmmSPInstruction.java | 42 +--
.../spark/QuantilePickSPInstruction.java | 34 +-
.../spark/QuantileSortSPInstruction.java | 34 +-
.../spark/QuaternarySPInstruction.java | 56 +--
.../instructions/spark/RandSPInstruction.java | 54 +--
.../spark/ReblockSPInstruction.java | 42 +--
.../spark/RelationalBinarySPInstruction.java | 20 +-
.../instructions/spark/ReorgSPInstruction.java | 46 +--
.../instructions/spark/RmmSPInstruction.java | 34 +-
.../instructions/spark/SPInstruction.java | 24 +-
.../spark/TernarySPInstruction.java | 50 +--
.../instructions/spark/TsmmSPInstruction.java | 24 +-
.../spark/UaggOuterChainSPInstruction.java | 56 +--
.../instructions/spark/UnarySPInstruction.java | 14 +-
.../instructions/spark/WriteSPInstruction.java | 42 +--
.../instructions/spark/ZipmmSPInstruction.java | 34 +-
.../spark/data/BlockPartitioner.java | 6 +-
.../spark/data/BroadcastObject.java | 2 +-
.../spark/data/CorrMatrixBlock.java | 8 +-
.../instructions/spark/data/CountLinesInfo.java | 2 +-
.../spark/data/LazyIterableIterator.java | 2 +-
.../instructions/spark/data/LineageObject.java | 4 +-
.../spark/data/PartitionedBroadcastMatrix.java | 8 +-
.../spark/data/PartitionedMatrixBlock.java | 22 +-
.../instructions/spark/data/RDDObject.java | 2 +-
.../instructions/spark/data/RDDProperties.java | 2 +-
.../instructions/spark/data/RowMatrixBlock.java | 8 +-
.../spark/data/SerLongWritable.java | 2 +-
.../instructions/spark/data/SerText.java | 2 +-
.../AggregateDropCorrectionFunction.java | 6 +-
.../ComputeBinaryBlockNnzFunction.java | 4 +-
.../ConvertColumnRDDToBinaryBlock.java | 10 +-
.../ConvertMLLibBlocksToBinaryBlocks.java | 8 +-
.../functions/ConvertMatrixBlockToIJVLines.java | 8 +-
.../functions/ConvertStringToLongTextPair.java | 2 +-
.../ConvertTextLineToBinaryCellFunction.java | 18 +-
.../spark/functions/ConvertTextToString.java | 2 +-
.../spark/functions/CopyBinaryCellFunction.java | 6 +-
.../spark/functions/CopyBlockFunction.java | 4 +-
.../spark/functions/CopyBlockPairFunction.java | 6 +-
.../spark/functions/CopyTextInputFunction.java | 2 +-
.../functions/ExtractBlockForBinaryReblock.java | 12 +-
.../spark/functions/ExtractGroup.java | 14 +-
.../spark/functions/ExtractGroupNWeights.java | 14 +-
.../functions/FilterDiagBlocksFunction.java | 6 +-
.../functions/FilterNonEmptyBlocksFunction.java | 6 +-
.../FindMatrixBlockFromMatrixIndexes.java | 6 +-
.../spark/functions/GetMIMBFromRow.java | 6 +-
.../spark/functions/GetMLBlock.java | 8 +-
.../spark/functions/GetMLLibBlocks.java | 12 +-
.../spark/functions/IsBlockInRange.java | 10 +-
.../spark/functions/LastCellInMatrixBlock.java | 6 +-
.../functions/MatrixMatrixBinaryOpFunction.java | 6 +-
.../functions/MatrixScalarUnaryFunction.java | 6 +-
.../functions/MatrixVectorBinaryOpFunction.java | 12 +-
.../MatrixVectorBinaryOpPartitionFunction.java | 14 +-
.../functions/OuterVectorBinaryOpFunction.java | 10 +-
.../functions/PerformGroupByAggInCombiner.java | 20 +-
.../functions/PerformGroupByAggInReducer.java | 20 +-
.../spark/functions/ReorgMapFunction.java | 16 +-
.../functions/ReplicateVectorFunction.java | 6 +-
.../spark/functions/SparkListener.java | 8 +-
.../UnflattenIterablesAfterCogroup.java | 6 +-
.../spark/utils/RDDAggregateUtils.java | 22 +-
.../spark/utils/RDDConverterUtils.java | 36 +-
.../spark/utils/RDDConverterUtilsExt.java | 32 +-
.../instructions/spark/utils/RDDSortUtils.java | 28 +-
.../instructions/spark/utils/SparkUtils.java | 24 +-
.../runtime/io/BinaryBlockSerialization.java | 10 +-
.../sysml/runtime/io/IOUtilFunctions.java | 4 +-
.../apache/sysml/runtime/io/MatrixReader.java | 10 +-
.../sysml/runtime/io/MatrixReaderFactory.java | 10 +-
.../apache/sysml/runtime/io/MatrixWriter.java | 8 +-
.../sysml/runtime/io/MatrixWriterFactory.java | 12 +-
.../apache/sysml/runtime/io/ReadProperties.java | 6 +-
.../sysml/runtime/io/ReaderBinaryBlock.java | 14 +-
.../runtime/io/ReaderBinaryBlockParallel.java | 14 +-
.../sysml/runtime/io/ReaderBinaryCell.java | 12 +-
.../apache/sysml/runtime/io/ReaderTextCSV.java | 14 +-
.../sysml/runtime/io/ReaderTextCSVParallel.java | 12 +-
.../apache/sysml/runtime/io/ReaderTextCell.java | 12 +-
.../runtime/io/ReaderTextCellParallel.java | 16 +-
.../sysml/runtime/io/WriterBinaryBlock.java | 20 +-
.../runtime/io/WriterBinaryBlockParallel.java | 20 +-
.../sysml/runtime/io/WriterBinaryCell.java | 20 +-
.../sysml/runtime/io/WriterMatrixMarket.java | 16 +-
.../runtime/io/WriterMatrixMarketParallel.java | 18 +-
.../apache/sysml/runtime/io/WriterTextCSV.java | 18 +-
.../sysml/runtime/io/WriterTextCSVParallel.java | 18 +-
.../apache/sysml/runtime/io/WriterTextCell.java | 16 +-
.../runtime/io/WriterTextCellParallel.java | 18 +-
.../apache/sysml/runtime/matrix/CMCOVMR.java | 20 +-
.../sysml/runtime/matrix/CSVReblockMR.java | 34 +-
.../apache/sysml/runtime/matrix/CleanupMR.java | 14 +-
.../apache/sysml/runtime/matrix/CombineMR.java | 40 +-
.../apache/sysml/runtime/matrix/DataGenMR.java | 56 +--
.../sysml/runtime/matrix/DataPartitionMR.java | 20 +-
.../org/apache/sysml/runtime/matrix/GMR.java | 66 ++--
.../sysml/runtime/matrix/GroupedAggMR.java | 24 +-
.../apache/sysml/runtime/matrix/JobReturn.java | 8 +-
.../org/apache/sysml/runtime/matrix/MMCJMR.java | 38 +-
.../org/apache/sysml/runtime/matrix/MMRJMR.java | 30 +-
.../runtime/matrix/MatrixCharacteristics.java | 84 ++---
.../matrix/MatrixDimensionsMetaData.java | 2 +-
.../runtime/matrix/MatrixFormatMetaData.java | 6 +-
.../apache/sysml/runtime/matrix/MetaData.java | 2 +-
.../apache/sysml/runtime/matrix/ReblockMR.java | 30 +-
.../org/apache/sysml/runtime/matrix/SortMR.java | 56 +--
.../apache/sysml/runtime/matrix/WriteCSVMR.java | 32 +-
.../matrix/data/AdaptivePartialBlock.java | 2 +-
.../matrix/data/AddDummyWeightConverter.java | 2 +-
.../data/BinaryBlockToBinaryCellConverter.java | 4 +-
.../data/BinaryBlockToRowBlockConverter.java | 6 +-
.../data/BinaryBlockToTextCellConverter.java | 4 +-
.../data/BinaryCellToRowBlockConverter.java | 2 +-
.../matrix/data/BinaryCellToTextConverter.java | 2 +-
.../sysml/runtime/matrix/data/CM_N_COVCell.java | 30 +-
.../matrix/data/CSVFileFormatProperties.java | 4 +-
.../sysml/runtime/matrix/data/CTableMap.java | 6 +-
.../sysml/runtime/matrix/data/Converter.java | 2 +-
.../matrix/data/FileFormatProperties.java | 2 +-
.../apache/sysml/runtime/matrix/data/IJV.java | 2 +-
.../runtime/matrix/data/IdenticalConverter.java | 2 +-
.../sysml/runtime/matrix/data/InputInfo.java | 8 +-
.../runtime/matrix/data/LibCommonsMath.java | 8 +-
.../sysml/runtime/matrix/data/LibMatrixAgg.java | 44 +--
.../runtime/matrix/data/LibMatrixBincell.java | 38 +-
.../runtime/matrix/data/LibMatrixDatagen.java | 14 +-
.../runtime/matrix/data/LibMatrixMult.java | 26 +-
.../runtime/matrix/data/LibMatrixOuterAgg.java | 38 +-
.../runtime/matrix/data/LibMatrixReorg.java | 22 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 100 ++---
.../matrix/data/MatrixBlockDataInput.java | 2 +-
.../matrix/data/MatrixBlockDataOutput.java | 2 +-
.../sysml/runtime/matrix/data/MatrixCell.java | 32 +-
.../runtime/matrix/data/MatrixIndexes.java | 4 +-
.../runtime/matrix/data/MatrixPackedCell.java | 12 +-
.../sysml/runtime/matrix/data/MatrixValue.java | 30 +-
.../matrix/data/MultipleOutputCommitter.java | 4 +-
.../data/NumItemsByEachReducerMetaData.java | 6 +-
.../matrix/data/OperationsOnMatrixValues.java | 32 +-
.../sysml/runtime/matrix/data/OutputInfo.java | 8 +-
.../apache/sysml/runtime/matrix/data/Pair.java | 2 +-
.../sysml/runtime/matrix/data/PartialBlock.java | 2 +-
.../data/PoissonRandomMatrixGenerator.java | 6 +-
.../matrix/data/RandomMatrixGenerator.java | 12 +-
.../runtime/matrix/data/RuntimeDataFormat.java | 2 +-
.../sysml/runtime/matrix/data/SparseRow.java | 4 +-
.../runtime/matrix/data/SparseRowsIterator.java | 2 +-
.../sysml/runtime/matrix/data/Tagged.java | 2 +-
.../matrix/data/TaggedAdaptivePartialBlock.java | 2 +-
.../matrix/data/TaggedFirstSecondIndexes.java | 8 +-
.../sysml/runtime/matrix/data/TaggedInt.java | 2 +-
.../runtime/matrix/data/TaggedMatrixBlock.java | 2 +-
.../runtime/matrix/data/TaggedMatrixCell.java | 2 +-
.../matrix/data/TaggedMatrixPackedCell.java | 2 +-
.../runtime/matrix/data/TaggedMatrixValue.java | 2 +-
.../runtime/matrix/data/TaggedPartialBlock.java | 2 +-
.../matrix/data/TaggedTripleIndexes.java | 4 +-
.../data/TextCellToRowBlockConverter.java | 4 +-
.../matrix/data/TextToBinaryCellConverter.java | 4 +-
.../runtime/matrix/data/TripleIndexes.java | 4 +-
.../matrix/data/UnPaddedOutputFormat.java | 2 +-
.../sysml/runtime/matrix/data/WeightedCell.java | 14 +-
.../data/WeightedCellToSortInputConverter.java | 2 +-
.../sysml/runtime/matrix/data/WeightedPair.java | 4 +-
.../data/WeightedPairToSortInputConverter.java | 2 +-
.../data/hadoopfix/DelegatingInputFormat.java | 2 +-
.../matrix/data/hadoopfix/MultipleInputs.java | 2 +-
.../matrix/data/hadoopfix/TaggedInputSplit.java | 2 +-
.../runtime/matrix/mapred/CMCOVMRMapper.java | 20 +-
.../runtime/matrix/mapred/CMCOVMRReducer.java | 28 +-
.../matrix/mapred/CSVAssignRowIDMapper.java | 14 +-
.../matrix/mapred/CSVAssignRowIDReducer.java | 6 +-
.../runtime/matrix/mapred/CSVReblockMapper.java | 18 +-
.../matrix/mapred/CSVReblockReducer.java | 20 +-
.../runtime/matrix/mapred/CSVWriteMapper.java | 12 +-
.../runtime/matrix/mapred/CSVWriteReducer.java | 20 +-
.../sysml/runtime/matrix/mapred/CachedMap.java | 2 +-
.../runtime/matrix/mapred/CachedMapElement.java | 2 +-
.../runtime/matrix/mapred/CachedValueMap.java | 6 +-
.../mapred/CollectMultipleConvertedOutputs.java | 8 +-
.../runtime/matrix/mapred/DataGenMapper.java | 18 +-
.../matrix/mapred/DistributedCacheInput.java | 18 +-
.../runtime/matrix/mapred/GMRCombiner.java | 10 +-
.../runtime/matrix/mapred/GMRCtableBuffer.java | 16 +-
.../sysml/runtime/matrix/mapred/GMRMapper.java | 14 +-
.../sysml/runtime/matrix/mapred/GMRReducer.java | 20 +-
.../matrix/mapred/GroupedAggMRCombiner.java | 22 +-
.../matrix/mapred/GroupedAggMRMapper.java | 14 +-
.../matrix/mapred/GroupedAggMRReducer.java | 24 +-
.../matrix/mapred/IndexedMatrixValue.java | 6 +-
.../runtime/matrix/mapred/MMCJMRCache.java | 12 +-
.../runtime/matrix/mapred/MMCJMRCombiner.java | 6 +-
.../mapred/MMCJMRCombinerReducerBase.java | 18 +-
.../runtime/matrix/mapred/MMCJMRInputCache.java | 8 +-
.../runtime/matrix/mapred/MMCJMRMapper.java | 14 +-
.../runtime/matrix/mapred/MMCJMRReducer.java | 20 +-
.../mapred/MMCJMRReducerWithAggregator.java | 22 +-
.../runtime/matrix/mapred/MMRJMRMapper.java | 14 +-
.../runtime/matrix/mapred/MMRJMRReducer.java | 20 +-
.../mapred/MRBaseForCommonInstructions.java | 48 +--
.../matrix/mapred/MRConfigurationNames.java | 2 +-
.../matrix/mapred/MRJobConfiguration.java | 108 +++---
.../sysml/runtime/matrix/mapred/MapperBase.java | 30 +-
.../matrix/mapred/PartialAggregator.java | 16 +-
.../runtime/matrix/mapred/ReblockBuffer.java | 18 +-
.../runtime/matrix/mapred/ReblockMapper.java | 22 +-
.../runtime/matrix/mapred/ReblockReducer.java | 18 +-
.../sysml/runtime/matrix/mapred/ReduceBase.java | 26 +-
.../operators/AggregateBinaryOperator.java | 8 +-
.../matrix/operators/AggregateOperator.java | 16 +-
.../operators/AggregateUnaryOperator.java | 14 +-
.../matrix/operators/BinaryOperator.java | 42 +--
.../runtime/matrix/operators/CMOperator.java | 6 +-
.../runtime/matrix/operators/COVOperator.java | 4 +-
.../matrix/operators/LeftScalarOperator.java | 16 +-
.../runtime/matrix/operators/Operator.java | 2 +-
.../matrix/operators/QuaternaryOperator.java | 20 +-
.../matrix/operators/ReIndexOperator.java | 2 +-
.../runtime/matrix/operators/ReorgOperator.java | 4 +-
.../matrix/operators/RightScalarOperator.java | 16 +-
.../matrix/operators/ScalarOperator.java | 32 +-
.../matrix/operators/SimpleOperator.java | 4 +-
.../runtime/matrix/operators/UnaryOperator.java | 6 +-
.../matrix/operators/ZeroOutOperator.java | 2 +-
.../sort/CompactDoubleIntInputFormat.java | 8 +-
.../runtime/matrix/sort/CompactInputFormat.java | 2 +-
.../matrix/sort/CompactOutputFormat.java | 2 +-
.../matrix/sort/IndexSortComparable.java | 2 +-
.../matrix/sort/IndexSortComparableDesc.java | 2 +-
.../runtime/matrix/sort/IndexSortMapper.java | 10 +-
.../runtime/matrix/sort/IndexSortReducer.java | 12 +-
.../matrix/sort/IndexSortStitchupMapper.java | 10 +-
.../matrix/sort/IndexSortStitchupReducer.java | 10 +-
.../matrix/sort/PickFromCompactInputFormat.java | 10 +-
.../runtime/matrix/sort/ReadWithZeros.java | 2 +-
.../matrix/sort/SamplingSortMRInputFormat.java | 10 +-
.../runtime/matrix/sort/ValueSortMapper.java | 16 +-
.../runtime/matrix/sort/ValueSortReducer.java | 6 +-
.../sysml/runtime/transform/ApplyTfBBMR.java | 36 +-
.../runtime/transform/ApplyTfBBMapper.java | 18 +-
.../sysml/runtime/transform/ApplyTfCSVMR.java | 12 +-
.../runtime/transform/ApplyTfCSVMapper.java | 4 +-
.../runtime/transform/ApplyTfCSVSPARK.java | 8 +-
.../sysml/runtime/transform/BinAgent.java | 6 +-
.../sysml/runtime/transform/DataTransform.java | 70 ++--
.../sysml/runtime/transform/DistinctValue.java | 6 +-
.../sysml/runtime/transform/DummycodeAgent.java | 4 +-
.../sysml/runtime/transform/GTFMTDMapper.java | 4 +-
.../sysml/runtime/transform/GTFMTDReducer.java | 6 +-
.../sysml/runtime/transform/GenTfMtdMR.java | 6 +-
.../sysml/runtime/transform/GenTfMtdSPARK.java | 8 +-
.../sysml/runtime/transform/MVImputeAgent.java | 20 +-
.../sysml/runtime/transform/OmitAgent.java | 4 +-
.../sysml/runtime/transform/RecodeAgent.java | 6 +-
.../apache/sysml/runtime/transform/TfUtils.java | 24 +-
.../runtime/transform/TransformationAgent.java | 2 +-
.../runtime/util/BinaryBlockInputFormat.java | 6 +-
.../runtime/util/BinaryBlockRecordReader.java | 6 +-
.../sysml/runtime/util/DataConverter.java | 38 +-
.../util/FastBufferedDataInputStream.java | 6 +-
.../util/FastBufferedDataOutputStream.java | 6 +-
.../sysml/runtime/util/FastStringTokenizer.java | 2 +-
.../apache/sysml/runtime/util/IndexRange.java | 2 +-
.../sysml/runtime/util/LocalFileUtils.java | 24 +-
.../runtime/util/LongLongDoubleHashMap.java | 2 +-
.../sysml/runtime/util/MapReduceTool.java | 30 +-
.../sysml/runtime/util/NormalPRNGenerator.java | 2 +-
.../apache/sysml/runtime/util/PRNGenerator.java | 2 +-
.../sysml/runtime/util/PoissonPRNGenerator.java | 2 +-
.../org/apache/sysml/runtime/util/RandN.java | 2 +-
.../apache/sysml/runtime/util/RandNPair.java | 2 +-
.../apache/sysml/runtime/util/SortUtils.java | 4 +-
.../sysml/runtime/util/UniformPRNGenerator.java | 2 +-
.../sysml/runtime/util/UtilFunctions.java | 8 +-
.../java/org/apache/sysml/udf/BinaryObject.java | 2 +-
.../ExternalFunctionInvocationInstruction.java | 10 +-
.../org/apache/sysml/udf/FunctionParameter.java | 2 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 28 +-
.../org/apache/sysml/udf/PackageFunction.java | 4 +-
.../sysml/udf/PackageRuntimeException.java | 2 +-
src/main/java/org/apache/sysml/udf/Scalar.java | 2 +-
.../apache/sysml/udf/lib/BinningWrapper.java | 16 +-
.../org/apache/sysml/udf/lib/DeNaNWrapper.java | 12 +-
.../sysml/udf/lib/DeNegInfinityWrapper.java | 12 +-
.../sysml/udf/lib/DynamicProjectMatrixCP.java | 18 +-
.../sysml/udf/lib/DynamicReadMatrixCP.java | 24 +-
.../sysml/udf/lib/DynamicReadMatrixRcCP.java | 26 +-
.../sysml/udf/lib/DynamicWriteMatrixCP.java | 24 +-
.../org/apache/sysml/udf/lib/GatherWrapper.java | 16 +-
.../org/apache/sysml/udf/lib/OrderWrapper.java | 14 +-
.../sysml/udf/lib/PermutationMatrixWrapper.java | 22 +-
.../apache/sysml/udf/lib/RemoveEmptyRows.java | 14 +-
.../org/apache/sysml/udf/lib/TimeWrapper.java | 18 +-
.../org/apache/sysml/utils/AppException.java | 4 +-
.../java/org/apache/sysml/utils/Explain.java | 86 ++---
.../InstallDependencyForIntegrationTests.java | 4 +-
.../java/org/apache/sysml/utils/JSONHelper.java | 2 +-
.../apache/sysml/utils/ParameterBuilder.java | 2 +-
.../java/org/apache/sysml/utils/Statistics.java | 20 +-
src/main/java/org/apache/sysml/utils/Timer.java | 2 +-
.../org/apache/sysml/yarn/DMLAppMaster.java | 12 +-
.../sysml/yarn/DMLAppMasterStatusReporter.java | 2 +-
.../apache/sysml/yarn/DMLAppMasterUtils.java | 32 +-
.../org/apache/sysml/yarn/DMLYarnClient.java | 14 +-
.../apache/sysml/yarn/DMLYarnClientProxy.java | 26 +-
.../apache/sysml/yarn/ropt/GridEnumeration.java | 8 +-
.../sysml/yarn/ropt/GridEnumerationEqui.java | 6 +-
.../sysml/yarn/ropt/GridEnumerationExp.java | 6 +-
.../sysml/yarn/ropt/GridEnumerationHybrid.java | 8 +-
.../sysml/yarn/ropt/GridEnumerationMemory.java | 22 +-
.../yarn/ropt/MRJobResourceInstruction.java | 4 +-
.../apache/sysml/yarn/ropt/ResourceConfig.java | 22 +-
.../sysml/yarn/ropt/ResourceOptimizer.java | 74 ++--
.../yarn/ropt/ResourceOptimizerCPMigration.java | 2 +-
.../sysml/yarn/ropt/YarnClusterAnalyzer.java | 6 +-
.../sysml/yarn/ropt/YarnClusterConfig.java | 2 +-
.../sysml/yarn/ropt/YarnOptimizerUtils.java | 8 +-
src/main/standalone/runStandaloneSystemML.bat | 2 +-
src/main/standalone/runStandaloneSystemML.sh | 2 +-
.../AutomatedScalabilityTestBase.java | 4 +-
.../test/integration/AutomatedTestBase.java | 42 +--
.../BinaryMatrixCharacteristics.java | 2 +-
.../test/integration/TestConfiguration.java | 4 +-
.../integration/TestMatrixCharacteristics.java | 2 +-
.../applications/ApplyTransformTest.java | 6 +-
.../integration/applications/ArimaTest.java | 10 +-
.../integration/applications/CsplineCGTest.java | 8 +-
.../integration/applications/CsplineDSTest.java | 8 +-
.../test/integration/applications/GLMTest.java | 12 +-
.../test/integration/applications/GNMFTest.java | 8 +-
.../test/integration/applications/HITSTest.java | 8 +-
.../test/integration/applications/ID3Test.java | 12 +-
.../integration/applications/L2SVMTest.java | 8 +-
.../applications/LinearLogRegTest.java | 8 +-
.../applications/LinearRegressionTest.java | 8 +-
.../applications/MDABivariateStatsTest.java | 10 +-
.../applications/MultiClassSVMTest.java | 8 +-
.../applications/NaiveBayesTest.java | 8 +-
.../integration/applications/PageRankTest.java | 8 +-
.../integration/applications/WelchTTest.java | 10 +-
.../BivariateCategoricalCategoricallTest.java | 10 +-
.../BivariateOrdinalOrdinalTest.java | 10 +-
.../BivariateScaleCategoricalTest.java | 10 +-
.../BivariateScaleScaleTest.java | 10 +-
.../descriptivestats/OrderStatisticsTest.java | 6 +-
.../UnivariateCategoricalTest.java | 8 +-
.../descriptivestats/UnivariateStatsBase.java | 12 +-
.../UnivariateUnweightedScaleDenseTest.java | 4 +-
.../UnivariateUnweightedScaleSparseTest.java | 4 +-
.../UnivariateWeightedScaleDenseTest.java | 4 +-
.../UnivariateWeightedScaleSparseTest.java | 4 +-
.../applications/dml/ApplyTransformDMLTest.java | 4 +-
.../applications/dml/ArimaDMLTest.java | 4 +-
.../applications/dml/CsplineCGDMLTest.java | 4 +-
.../applications/dml/CsplineDSDMLTest.java | 4 +-
.../applications/dml/GLMDMLTest.java | 4 +-
.../applications/dml/GNMFDMLTest.java | 4 +-
.../applications/dml/HITSDMLTest.java | 4 +-
.../applications/dml/ID3DMLTest.java | 4 +-
.../applications/dml/L2SVMDMLTest.java | 4 +-
.../applications/dml/LinearLogRegDMLTest.java | 4 +-
.../dml/LinearRegressionDMLTest.java | 4 +-
.../dml/MDABivariateStatsDMLTest.java | 4 +-
.../applications/dml/MultiClassSVMDMLTest.java | 4 +-
.../applications/dml/NaiveBayesDMLTest.java | 4 +-
.../applications/dml/PageRankDMLTest.java | 4 +-
.../applications/dml/WelchTDMLTest.java | 4 +-
.../parfor/ParForBivariateStatsTest.java | 16 +-
.../parfor/ParForCVMulticlassSVMTest.java | 12 +-
.../parfor/ParForCorrelationTest.java | 18 +-
.../parfor/ParForCorrelationTestLarge.java | 14 +-
.../parfor/ParForNaiveBayesTest.java | 20 +-
.../parfor/ParForUnivariateStatsTest.java | 18 +-
.../pydml/ApplyTransformPyDMLTest.java | 4 +-
.../applications/pydml/ArimaPyDMLTest.java | 4 +-
.../applications/pydml/CsplineCGPyDMLTest.java | 4 +-
.../applications/pydml/CsplineDSPyDMLTest.java | 4 +-
.../applications/pydml/GLMPyDMLTest.java | 4 +-
.../applications/pydml/GNMFPyDMLTest.java | 4 +-
.../applications/pydml/HITSPyDMLTest.java | 4 +-
.../applications/pydml/ID3PyDMLTest.java | 4 +-
.../applications/pydml/L2SVMPyDMLTest.java | 4 +-
.../pydml/LinearLogRegPyDMLTest.java | 4 +-
.../pydml/LinearRegressionPyDMLTest.java | 4 +-
.../pydml/MDABivariateStatsPyDMLTest.java | 4 +-
.../pydml/MultiClassSVMPyDMLTest.java | 4 +-
.../applications/pydml/NaiveBayesPyDMLTest.java | 4 +-
.../applications/pydml/PageRankPyDMLTest.java | 4 +-
.../applications/pydml/WelchTPyDMLTest.java | 4 +-
.../functions/aggregate/AggregateInfTest.java | 14 +-
.../functions/aggregate/ColSumTest.java | 8 +-
.../functions/aggregate/ColSumsSqTest.java | 24 +-
.../functions/aggregate/FullAggregateTest.java | 18 +-
.../aggregate/FullColAggregateTest.java | 18 +-
.../aggregate/FullGroupedAggregateTest.java | 24 +-
.../aggregate/FullRowAggregateTest.java | 18 +-
.../functions/aggregate/LengthTest.java | 8 +-
.../functions/aggregate/MaxTest.java | 8 +-
.../functions/aggregate/MinTest.java | 8 +-
.../functions/aggregate/NColTest.java | 8 +-
.../functions/aggregate/NRowTest.java | 8 +-
.../functions/aggregate/ProdTest.java | 8 +-
.../functions/aggregate/RowSumTest.java | 8 +-
.../functions/aggregate/RowSumsSqTest.java | 24 +-
.../functions/aggregate/SumSqTest.java | 24 +-
.../functions/aggregate/SumTest.java | 8 +-
.../functions/aggregate/TraceTest.java | 8 +-
.../functions/append/AppendChainTest.java | 16 +-
.../functions/append/AppendMatrixTest.java | 22 +-
.../functions/append/AppendVectorTest.java | 18 +-
.../functions/append/RBindCBindMatrixTest.java | 22 +-
.../functions/append/StringAppendTest.java | 16 +-
.../binary/matrix/BinUaggChainTest.java | 16 +-
.../binary/matrix/CentralMomentTest.java | 16 +-
.../functions/binary/matrix/CovarianceTest.java | 16 +-
.../matrix/DiagMatrixMultiplicationTest.java | 18 +-
.../ElementwiseAdditionMultiplicationTest.java | 6 +-
.../binary/matrix/ElementwiseAdditionTest.java | 8 +-
.../binary/matrix/ElementwiseDivisionTest.java | 8 +-
.../binary/matrix/ElementwiseModulusTest.java | 10 +-
.../matrix/ElementwiseMultiplicationTest.java | 10 +-
.../matrix/ElementwiseSubtractionTest.java | 8 +-
.../binary/matrix/MapMultChainTest.java | 22 +-
.../binary/matrix/MapMultLimitTest.java | 12 +-
.../binary/matrix/MatrixMultiplicationTest.java | 12 +-
.../binary/matrix/MatrixVectorTest.java | 10 +-
.../binary/matrix/OuterProductTest.java | 14 +-
.../functions/binary/matrix/QuantileTest.java | 16 +-
.../binary/matrix/ScalarAdditionTest.java | 6 +-
.../binary/matrix/ScalarDivisionTest.java | 8 +-
.../binary/matrix/ScalarModulusTest.java | 8 +-
.../binary/matrix/ScalarMultiplicationTest.java | 8 +-
.../binary/matrix/ScalarSubtractionTest.java | 6 +-
.../TransposeMatrixMultiplicationTest.java | 16 +-
.../binary/matrix/UaggOuterChainTest.java | 24 +-
.../UltraSparseMRMatrixMultiplicationTest.java | 20 +-
.../ZipMMSparkMatrixMultiplicationTest.java | 20 +-
.../FullMatrixMatrixCellwiseOperationTest.java | 16 +-
...ullMatrixVectorColCellwiseOperationTest.java | 16 +-
...ullMatrixVectorRowCellwiseOperationTest.java | 16 +-
.../FullMinus1MultTest.java | 20 +-
...ectorVectorCellwiseCompareOperationTest.java | 18 +-
.../FullVectorVectorCellwiseOperationTest.java | 18 +-
...FullDistributedMatrixMultiplicationTest.java | 20 +-
.../FullIntegerDivisionTest.java | 24 +-
.../FullMatrixMultiplicationTest.java | 14 +-
...llMatrixMultiplicationTransposeSelfTest.java | 16 +-
...FullMatrixMultiplicationUltraSparseTest.java | 14 +-
.../FullMinMaxComparisonTest.java | 24 +-
.../matrix_full_other/FullPPredMatrixTest.java | 16 +-
.../FullPPredScalarLeftTest.java | 14 +-
.../FullPPredScalarRightTest.java | 14 +-
.../binary/matrix_full_other/FullPowerTest.java | 24 +-
.../functions/binary/scalar/AdditionTest.java | 8 +-
.../functions/binary/scalar/AndTest.java | 6 +-
.../functions/binary/scalar/DivisionTest.java | 6 +-
.../functions/binary/scalar/EqualTest.java | 6 +-
.../binary/scalar/FullStringComparisonTest.java | 8 +-
.../binary/scalar/GreaterThanOrEqualTest.java | 6 +-
.../binary/scalar/GreaterThanTest.java | 8 +-
.../binary/scalar/LessThanOrEqualTest.java | 6 +-
.../functions/binary/scalar/LessThanTest.java | 8 +-
.../functions/binary/scalar/LogarithmTest.java | 6 +-
.../functions/binary/scalar/ModulusTest.java | 8 +-
.../binary/scalar/MultiplicationTest.java | 6 +-
.../functions/binary/scalar/OrTest.java | 6 +-
.../functions/binary/scalar/PowerTest.java | 6 +-
.../binary/scalar/SubtractionTest.java | 6 +-
.../functions/blocks/VariableTest.java | 8 +-
.../integration/functions/blocks/WhileTest.java | 6 +-
.../caching/CachingPWriteExportTest.java | 16 +-
.../functions/data/FullReblockTest.java | 30 +-
.../data/FullStringInitializeTest.java | 22 +-
.../functions/data/RandRuntimePlatformTest.java | 14 +-
.../integration/functions/data/RandTest1.java | 6 +-
.../integration/functions/data/RandTest2.java | 6 +-
.../integration/functions/data/RandTest3.java | 6 +-
.../integration/functions/data/RandTest4.java | 6 +-
.../integration/functions/data/RandTest5.java | 14 +-
.../functions/data/RandVarMinMaxTest.java | 16 +-
.../functions/data/RandVarSeedTest.java | 24 +-
.../integration/functions/data/ReadMMTest.java | 20 +-
.../integration/functions/data/ReblockTest.java | 8 +-
.../integration/functions/data/SampleTest.java | 12 +-
.../functions/data/SequenceTest.java | 14 +-
.../functions/data/VariableTest.java | 6 +-
.../integration/functions/data/WriteMMTest.java | 16 +-
.../integration/functions/data/WriteTest.java | 12 +-
.../functions/dmlscript/DMLScriptTest1.java | 6 +-
.../functions/dmlscript/DMLScriptTest2.java | 8 +-
.../functions/external/DynProjectTest.java | 10 +-
.../functions/external/DynReadWriteTest.java | 12 +-
.../external/FunctionExpressionsTest.java | 12 +-
.../functions/external/OrderTest.java | 12 +-
.../functions/gdfo/GDFOLinregCG.java | 16 +-
.../functions/gdfo/GDFOLinregDS.java | 16 +-
.../functions/gdfo/GDFOLinregDSsimpl.java | 16 +-
.../functions/gdfo/GDFOMMChainLoop.java | 14 +-
.../gdfo/HashInterestingPropertiesTest.java | 14 +-
.../indexing/Jdk7IssueRightIndexingTest.java | 14 +-
.../indexing/LeftIndexingScalarTest.java | 16 +-
.../indexing/LeftIndexingSparseDenseTest.java | 20 +-
.../indexing/LeftIndexingSparseSparseTest.java | 20 +-
.../functions/indexing/LeftIndexingTest.java | 20 +-
.../indexing/RightIndexingMatrixTest.java | 16 +-
.../indexing/RightIndexingVectorTest.java | 16 +-
.../functions/io/FullDynWriteTest.java | 30 +-
.../test/integration/functions/io/IOTest1.java | 6 +-
.../test/integration/functions/io/IOTest2.java | 8 +-
.../test/integration/functions/io/IOTest3.java | 8 +-
.../test/integration/functions/io/IOTest4.java | 8 +-
.../test/integration/functions/io/IOTest5.java | 8 +-
.../integration/functions/io/ScalarIOTest.java | 10 +-
.../functions/io/SeqParReadTest.java | 24 +-
.../functions/io/binary/SerializeTest.java | 22 +-
.../functions/io/csv/CSVParametersTest.java | 12 +-
.../functions/io/csv/FormatChangeTest.java | 12 +-
.../functions/io/csv/ReadCSVTest.java | 12 +-
.../functions/io/csv/WriteCSVTest.java | 10 +-
.../io/matrixmarket/FormatChangeTest.java | 10 +-
.../functions/io/matrixmarket/ReadMMTest.java | 12 +-
.../jmlc/SystemTMulticlassSVMScoreTest.java | 26 +-
.../functions/misc/ConditionalValidateTest.java | 22 +-
.../functions/misc/DataTypeCastingTest.java | 14 +-
.../functions/misc/DataTypeChangeTest.java | 22 +-
.../functions/misc/FunctionInliningTest.java | 12 +-
.../misc/IPALiteralReplacementTest.java | 12 +-
.../functions/misc/IPAScalarRecursionTest.java | 6 +-
.../functions/misc/IPAUnknownRecursionTest.java | 12 +-
.../misc/InvalidFunctionSignatureTest.java | 8 +-
.../functions/misc/LongOverflowTest.java | 10 +-
.../functions/misc/NrowNcolStringTest.java | 6 +-
.../misc/NrowNcolUnknownCSVReadTest.java | 20 +-
.../functions/misc/OuterTableExpandTest.java | 20 +-
.../functions/misc/PrintExpressionTest.java | 10 +-
.../functions/misc/PrintMatrixTest.java | 8 +-
.../functions/misc/ReadAfterWriteTest.java | 10 +-
.../RewriteSimplifyRowColSumMVMultTest.java | 14 +-
.../misc/RewriteSlicedMatrixMultTest.java | 12 +-
.../functions/misc/ScalarAssignmentTest.java | 10 +-
.../functions/misc/ScalarFunctionTest.java | 12 +-
.../functions/misc/SetWorkingDirTest.java | 10 +-
.../misc/ValueTypeAutoCastingTest.java | 10 +-
.../functions/misc/ValueTypeCastingTest.java | 16 +-
.../functions/mlcontext/GNMFTest.java | 32 +-
.../functions/parfor/ForLoopPredicateTest.java | 8 +-
.../parfor/ParForAdversarialLiteralsTest.java | 12 +-
.../ParForColwiseDataPartitioningTest.java | 20 +-
.../ParForDataPartitionLeftIndexingTest.java | 14 +-
.../parfor/ParForDependencyAnalysisTest.java | 20 +-
.../parfor/ParForFunctionSerializationTest.java | 10 +-
.../ParForMultipleDataPartitioningTest.java | 12 +-
.../parfor/ParForNaNResultMergeTest.java | 10 +-
.../ParForParallelRemoteResultMergeTest.java | 12 +-
.../parfor/ParForRepeatedOptimizationTest.java | 22 +-
.../ParForReplaceThreadIDRecompileTest.java | 12 +-
.../ParForRowwiseDataPartitioningTest.java | 20 +-
.../parfor/ParForRulebasedOptimizerTest.java | 10 +-
.../ParForSerialRemoteResultMergeTest.java | 16 +-
.../piggybacking/PiggybackingTest1.java | 14 +-
.../piggybacking/PiggybackingTest2.java | 12 +-
.../quaternary/RewritesWeightedSigmoidTest.java | 26 +-
.../quaternary/WeightedCrossEntropyTest.java | 30 +-
.../quaternary/WeightedDivMatrixMultTest.java | 30 +-
.../quaternary/WeightedSigmoidTest.java | 26 +-
.../quaternary/WeightedSquaredLossTest.java | 30 +-
.../quaternary/WeightedUnaryMatrixMultTest.java | 30 +-
.../functions/recompile/BranchRemovalTest.java | 12 +-
.../recompile/CSVReadUnknownSizeTest.java | 26 +-
.../recompile/FunctionRecompileTest.java | 14 +-
.../IPAAssignConstantPropagationTest.java | 12 +-
.../recompile/IPAComplexAppendTest.java | 22 +-
.../recompile/IPAConstantPropagationTest.java | 12 +-
...IPAPropagationSizeMultipleFunctionsTest.java | 12 +-
.../LiteralReplaceCastScalarReadTest.java | 14 +-
.../recompile/MultipleReadsIPATest.java | 16 +-
.../recompile/PredicateRecompileTest.java | 12 +-
.../recompile/RandJobRecompileTest.java | 12 +-
.../functions/recompile/RandRecompileTest.java | 10 +-
.../recompile/RandSizeExpressionEvalTest.java | 12 +-
.../recompile/ReblockRecompileTest.java | 22 +-
.../recompile/RemoveEmptyPotpourriTest.java | 12 +-
.../recompile/RemoveEmptyRecompileTest.java | 14 +-
.../RewriteComplexMapMultChainTest.java | 16 +-
.../SparsityFunctionRecompileTest.java | 26 +-
.../recompile/SparsityRecompileTest.java | 28 +-
.../functions/reorg/DiagV2MTest.java | 14 +-
.../functions/reorg/FullOrderTest.java | 20 +-
.../functions/reorg/FullTransposeTest.java | 16 +-
.../functions/reorg/MatrixReshapeTest.java | 16 +-
.../terms/ScalarMatrixUnaryBinaryTermTest.java | 8 +-
.../terms/ScalarToMatrixInLoopTest.java | 6 +-
.../ternary/CTableMatrixIgnoreZerosTest.java | 18 +-
.../functions/ternary/CTableSequenceTest.java | 18 +-
.../ternary/CentralMomentWeightsTest.java | 16 +-
.../ternary/CovarianceWeightsTest.java | 16 +-
.../functions/ternary/QuantileWeightsTest.java | 16 +-
.../functions/ternary/TableOutputTest.java | 14 +-
.../functions/transform/RunTest.java | 14 +-
.../functions/transform/ScalingTest.java | 28 +-
.../functions/transform/TransformTest.java | 24 +-
.../functions/unary/matrix/ACosTest.java | 8 +-
.../functions/unary/matrix/ASinTest.java | 6 +-
.../functions/unary/matrix/ATanTest.java | 8 +-
.../functions/unary/matrix/AbsTest.java | 6 +-
.../unary/matrix/CastAsScalarTest.java | 6 +-
.../functions/unary/matrix/CosTest.java | 6 +-
.../functions/unary/matrix/DiagTest.java | 8 +-
.../unary/matrix/EigenFactorizeTest.java | 10 +-
.../functions/unary/matrix/FullCummaxTest.java | 20 +-
.../functions/unary/matrix/FullCumminTest.java | 20 +-
.../functions/unary/matrix/FullCumprodTest.java | 20 +-
.../functions/unary/matrix/FullCumsumTest.java | 20 +-
.../unary/matrix/FullSelectPosTest.java | 22 +-
.../functions/unary/matrix/IQMTest.java | 10 +-
.../functions/unary/matrix/LUFactorizeTest.java | 10 +-
.../unary/matrix/MLUnaryBuiltinTest.java | 18 +-
.../unary/matrix/MatrixInverseTest.java | 10 +-
.../functions/unary/matrix/MinusTest.java | 14 +-
.../functions/unary/matrix/NegationTest.java | 6 +-
.../functions/unary/matrix/PrintTest.java | 6 +-
.../functions/unary/matrix/QRSolverTest.java | 10 +-
.../functions/unary/matrix/RemoveEmptyTest.java | 16 +-
.../functions/unary/matrix/ReplaceTest.java | 18 +-
.../functions/unary/matrix/RoundTest.java | 10 +-
.../functions/unary/matrix/SinTest.java | 6 +-
.../functions/unary/matrix/SqrtTest.java | 10 +-
.../functions/unary/matrix/TanTest.java | 6 +-
.../functions/unary/matrix/TransposeTest.java | 8 +-
.../functions/unary/scalar/ACosTest.java | 8 +-
.../functions/unary/scalar/ASinTest.java | 8 +-
.../functions/unary/scalar/ATanTest.java | 8 +-
.../functions/unary/scalar/AbsTest.java | 8 +-
.../functions/unary/scalar/BooleanTest.java | 6 +-
.../functions/unary/scalar/CosTest.java | 8 +-
.../functions/unary/scalar/ExponentTest.java | 10 +-
.../unary/scalar/FullDistributionTest.java | 10 +-
.../functions/unary/scalar/NegationTest.java | 8 +-
.../functions/unary/scalar/NotTest.java | 6 +-
.../functions/unary/scalar/PrintTest.java | 6 +-
.../functions/unary/scalar/RoundTest.java | 10 +-
.../functions/unary/scalar/SinTest.java | 8 +-
.../functions/unary/scalar/SqrtTest.java | 8 +-
.../functions/unary/scalar/StopTest.java | 6 +-
.../functions/unary/scalar/StopTest2.java | 6 +-
.../functions/unary/scalar/StopTestCtrlStr.java | 8 +-
.../functions/unary/scalar/TanTest.java | 8 +-
.../functions/vect/AutoVectorizationTest.java | 10 +-
.../scalability/LinearRegressionTest.java | 8 +-
.../integration/scalability/PageRankTest.java | 6 +-
.../org/apache/sysml/test/utils/TestUtils.java | 18 +-
.../applications/ctableStats/stratstats.dml | 2 +-
.../functions/aggregate/ColSumScalarTest.dml | 2 +-
.../scripts/functions/aggregate/ColSumTest.dml | 2 +-
.../functions/aggregate/LengthScalarTest.dml | 2 +-
.../scripts/functions/aggregate/LengthTest.dml | 2 +-
.../functions/aggregate/MaxScalarTest.dml | 2 +-
.../scripts/functions/aggregate/MaxTest.dml | 2 +-
.../functions/aggregate/MinScalarTest.dml | 2 +-
.../scripts/functions/aggregate/MinTest.dml | 2 +-
.../functions/aggregate/NColScalarTest.dml | 2 +-
.../scripts/functions/aggregate/NColTest.dml | 2 +-
.../functions/aggregate/NRowScalarTest.dml | 2 +-
.../scripts/functions/aggregate/NRowTest.dml | 2 +-
.../functions/aggregate/ProdScalarTest.dml | 2 +-
.../scripts/functions/aggregate/ProdTest.dml | 2 +-
.../functions/aggregate/RowSumScalarTest.dml | 2 +-
.../scripts/functions/aggregate/RowSumTest.dml | 2 +-
.../functions/aggregate/SumScalarTest.dml | 2 +-
.../scripts/functions/aggregate/SumTest.dml | 2 +-
.../functions/aggregate/TraceScalarTest.dml | 2 +-
.../scripts/functions/aggregate/TraceTest.dml | 2 +-
.../binary/matrix/AMultASpecial1Test.dml | 2 +-
.../binary/matrix/AMultBSpecial2Test.dml | 2 +-
.../ElementwiseAdditionMultiplicationTerm1.dml | 2 +-
.../ElementwiseAdditionMultiplicationTerm2.dml | 2 +-
.../ElementwiseAdditionMultiplicationTerm3.dml | 2 +-
.../binary/matrix/ElementwiseAdditionTest.dml | 2 +-
...lementwiseAdditionVariableDimensionsTest.dml | 2 +-
.../binary/matrix/ElementwiseDivisionTest.dml | 2 +-
...lementwiseDivisionVariableDimensionsTest.dml | 2 +-
.../binary/matrix/ElementwiseModulusTest.dml | 2 +-
...ElementwiseModulusVariableDimensionsTest.dml | 2 +-
.../matrix/ElementwiseMultiplicationTest.dml | 2 +-
...wiseMultiplicationVariableDimensionsTest.dml | 2 +-
.../matrix/ElementwiseSubtractionTest.dml | 2 +-
...entwiseSubtractionVariableDimensionsTest.dml | 2 +-
.../binary/matrix/MatrixMultiplicationTest.dml | 2 +-
.../matrix/MatrixVectorMultiplication.dml | 2 +-
.../binary/matrix/ScalarAdditionTest.dml | 2 +-
.../binary/matrix/ScalarDivisionTest.dml | 2 +-
.../binary/matrix/ScalarModulusTest.dml | 2 +-
.../binary/matrix/ScalarMultiplicationTest.dml | 2 +-
.../binary/matrix/ScalarSubtractionTest.dml | 2 +-
.../matrix/UltraSparseMatrixMultiplication.dml | 2 +-
.../matrix/UltraSparseMatrixMultiplication2.dml | 2 +-
.../FullMatrixCellwiseOperation_Addition.dml | 2 +-
.../FullMatrixCellwiseOperation_Division.dml | 2 +-
...llMatrixCellwiseOperation_Multiplication.dml | 2 +-
...FullMatrixCellwiseOperation_Substraction.dml | 2 +-
...atrixVectorColCellwiseOperation_Addition.dml | 2 +-
...atrixVectorColCellwiseOperation_Division.dml | 2 +-
...ectorColCellwiseOperation_Multiplication.dml | 2 +-
...xVectorColCellwiseOperation_Substraction.dml | 2 +-
...atrixVectorRowCellwiseOperation_Addition.dml | 2 +-
...atrixVectorRowCellwiseOperation_Division.dml | 2 +-
...ectorRowCellwiseOperation_Multiplication.dml | 2 +-
...xVectorRowCellwiseOperation_Substraction.dml | 2 +-
.../FullMatrixMultiplication.dml | 2 +-
.../binary/matrix_full_other/FullPower.dml | 2 +-
.../matrix_full_other/IntegerDivision_div.dml | 2 +-
.../matrix_full_other/IntegerDivision_mod.dml | 2 +-
.../TransposeSelfMatrixMultiplication1.dml | 2 +-
.../TransposeSelfMatrixMultiplication2.dml | 2 +-
.../binary/scalar/AdditionSingleTest.dml | 2 +-
.../functions/binary/scalar/AdditionTest.dml | 2 +-
.../scripts/functions/binary/scalar/AndTest.dml | 2 +-
.../binary/scalar/DivisionSingleTest.dml | 2 +-
.../functions/binary/scalar/DivisionTest.dml | 2 +-
.../functions/binary/scalar/EqualTest.dml | 2 +-
.../binary/scalar/GreaterThanOrEqualTest.dml | 2 +-
.../functions/binary/scalar/GreaterThanTest.dml | 2 +-
.../binary/scalar/LessThanOrEqualTest.dml | 2 +-
.../functions/binary/scalar/LessThanTest.dml | 2 +-
.../functions/binary/scalar/LogarithmTest.dml | 2 +-
.../binary/scalar/ModulusSingleTest.dml | 2 +-
.../functions/binary/scalar/ModulusTest.dml | 2 +-
.../binary/scalar/MultiplicationTest.dml | 2 +-
.../scripts/functions/binary/scalar/OrTest.dml | 2 +-
.../functions/binary/scalar/PowerTest.dml | 2 +-
.../scalar/SubtractionMultipleOperantsTest.dml | 2 +-
.../functions/binary/scalar/SubtractionTest.dml | 2 +-
.../functions/blocks/VariableAnalysis1Test.dml | 2 +-
.../functions/blocks/VariablePassing1Test.dml | 2 +-
.../functions/blocks/VariablePassing2Test.dml | 2 +-
src/test/scripts/functions/blocks/WhileTest.dml | 2 +-
.../scripts/functions/data/CopyVariableTest.dml | 2 +-
.../scripts/functions/data/RandScalarTest.dml | 2 +-
src/test/scripts/functions/data/RandTest1.dml | 2 +-
src/test/scripts/functions/data/RandTest2.dml | 2 +-
src/test/scripts/functions/data/RandTest3.dml | 2 +-
src/test/scripts/functions/data/RandTest4.dml | 2 +-
.../scripts/functions/data/ReadMMIndexTest.dml | 2 +-
src/test/scripts/functions/data/ReadMMTest.dml | 2 +-
src/test/scripts/functions/data/ReblockTest.dml | 2 +-
.../functions/data/WriteMMComplexTest.dml | 2 +-
src/test/scripts/functions/data/WriteMMTest.dml | 2 +-
src/test/scripts/functions/data/WriteTest.dml | 2 +-
.../scripts/functions/data/WriteTwiceTest.dml | 2 +-
.../functions/dmlscript/DMLScriptTest.dml | 2 +-
.../scripts/functions/external/DynProject.dml | 2 +-
.../scripts/functions/external/DynReadWrite.dml | 4 +-
.../functions/external/FunctionExpressions1.dml | 2 +-
.../functions/external/FunctionExpressions2.dml | 2 +-
src/test/scripts/functions/external/Order1.dml | 2 +-
src/test/scripts/functions/external/Order2.dml | 2 +-
src/test/scripts/functions/external/Outlier.dml | 2 +-
.../scripts/functions/external/SGDMFTest.dml | 2 +-
.../scripts/functions/external/SeqMiner.dml | 2 +-
src/test/scripts/functions/external/kMeans.dml | 2 +-
src/test/scripts/functions/external/kMeans2.dml | 4 +-
src/test/scripts/functions/io/IOTest1.dml | 2 +-
src/test/scripts/functions/io/IOTest2.dml | 2 +-
src/test/scripts/functions/io/IOTest3.dml | 2 +-
src/test/scripts/functions/io/IOTest4.dml | 2 +-
src/test/scripts/functions/io/IOTest5.dml | 2 +-
src/test/scripts/functions/parfor/parfor50.dml | 2 +-
.../functions/parfor/parfor_extfunct.dml | 4 +-
.../functions/parfor/parfor_literals1a.dml | 2 +-
.../functions/parfor/parfor_literals1b.dml | 2 +-
.../functions/parfor/parfor_literals1c.dml | 2 +-
.../functions/parfor/parfor_literals2.dml | 2 +-
.../terms/TestScalarToMatrixInLoop.dml | 2 +-
src/test/scripts/functions/terms/TestTerm1.dml | 2 +-
.../scripts/functions/unary/matrix/ACosTest.dml | 2 +-
.../scripts/functions/unary/matrix/ASinTest.dml | 2 +-
.../scripts/functions/unary/matrix/ATanTest.dml | 2 +-
.../scripts/functions/unary/matrix/AbsTest.dml | 2 +-
.../functions/unary/matrix/CastAsScalarTest.dml | 2 +-
.../scripts/functions/unary/matrix/CosTest.dml | 2 +-
.../functions/unary/matrix/DiagSingleTest.dml | 2 +-
.../scripts/functions/unary/matrix/DiagTest.dml | 2 +-
.../functions/unary/matrix/NegationTest.dml | 2 +-
.../scripts/functions/unary/matrix/QRsolve.dml | 2 +-
.../scripts/functions/unary/matrix/SinTest.dml | 2 +-
.../functions/unary/matrix/SqrtSingleTest.dml | 2 +-
.../scripts/functions/unary/matrix/SqrtTest.dml | 2 +-
.../scripts/functions/unary/matrix/TanTest.dml | 2 +-
.../functions/unary/matrix/TransposeTest.dml | 2 +-
.../scripts/functions/unary/scalar/ACosTest.dml | 2 +-
.../scripts/functions/unary/scalar/ASinTest.dml | 2 +-
.../scripts/functions/unary/scalar/ATanTest.dml | 2 +-
.../scripts/functions/unary/scalar/AbsTest.dml | 2 +-
.../functions/unary/scalar/BooleanWhileTest.dml | 2 +-
.../scripts/functions/unary/scalar/CosTest.dml | 2 +-
.../unary/scalar/ExponentBinaryTest.dml | 2 +-
.../functions/unary/scalar/ExponentTest.dml | 2 +-
.../functions/unary/scalar/NegationTest.dml | 2 +-
.../scripts/functions/unary/scalar/NotTest.dml | 2 +-
.../functions/unary/scalar/PrintTest.dml | 2 +-
.../functions/unary/scalar/PrintTest2.dml | 2 +-
.../scripts/functions/unary/scalar/SinTest.dml | 2 +-
.../functions/unary/scalar/SqrtSingleTest.dml | 2 +-
.../scripts/functions/unary/scalar/SqrtTest.dml | 2 +-
.../functions/unary/scalar/StopTest1.dml | 2 +-
.../functions/unary/scalar/StopTest2.dml | 2 +-
.../functions/unary/scalar/StopTestLoops_fn.dml | 2 +-
.../unary/scalar/StopTestLoops_for.dml | 2 +-
.../unary/scalar/StopTestLoops_parfor.dml | 2 +-
.../unary/scalar/StopTestLoops_while.dml | 2 +-
.../scripts/functions/unary/scalar/TanTest.dml | 2 +-
.../integration/applications/ZPackageSuite.java | 66 ++--
.../descriptivestats/ZPackageSuite.java | 2 +-
.../applications/parfor/ZPackageSuite.java | 2 +-
.../functions/aggregate/ZPackageSuite.java | 2 +-
.../functions/append/ZPackageSuite.java | 2 +-
.../functions/binary/matrix/ZPackageSuite.java | 2 +-
.../matrix_full_cellwise/ZPackageSuite.java | 2 +-
.../binary/matrix_full_other/ZPackageSuite.java | 2 +-
.../functions/binary/scalar/ZPackageSuite.java | 2 +-
.../functions/blocks/ZPackageSuite.java | 2 +-
.../functions/caching/ZPackageSuite.java | 2 +-
.../functions/data/ZPackageSuite.java | 2 +-
.../functions/dmlscript/ZPackageSuite.java | 2 +-
.../functions/external/ZPackageSuite.java | 2 +-
.../functions/gdfo/ZPackageSuite.java | 2 +-
.../functions/indexing/ZPackageSuite.java | 2 +-
.../integration/functions/io/ZPackageSuite.java | 2 +-
.../functions/io/binary/ZPackageSuite.java | 2 +-
.../functions/io/csv/ZPackageSuite.java | 2 +-
.../io/matrixmarket/ZPackageSuite.java | 2 +-
.../functions/jmlc/ZPackageSuite.java | 2 +-
.../functions/misc/ZPackageSuite.java | 2 +-
.../functions/mlcontext/ZPackageSuite.java | 2 +-
.../functions/parfor/ZPackageSuite.java | 2 +-
.../functions/piggybacking/ZPackageSuite.java | 2 +-
.../functions/quaternary/ZPackageSuite.java | 2 +-
.../functions/recompile/ZPackageSuite.java | 2 +-
.../functions/reorg/ZPackageSuite.java | 2 +-
.../functions/terms/ZPackageSuite.java | 2 +-
.../functions/ternary/ZPackageSuite.java | 2 +-
.../functions/transform/ZPackageSuite.java | 2 +-
.../functions/unary/matrix/ZPackageSuite.java | 2 +-
.../functions/unary/scalar/ZPackageSuite.java | 2 +-
.../functions/vect/ZPackageSuite.java | 2 +-
.../integration/scalability/ZPackageSuite.java | 2 +-
1392 files changed, 10256 insertions(+), 10256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 5e54ba1..d59cc60 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,14 +37,14 @@ src/main/java/Dml.tokens
src/main/java/DmlLexer.tokens
src/main/java/Pydml.tokens
src/main/java/PydmlLexer.tokens
-src/main/java/com/ibm/bi/dml/parser/antlr4/DmlBaseListener.java
-src/main/java/com/ibm/bi/dml/parser/antlr4/DmlLexer.java
-src/main/java/com/ibm/bi/dml/parser/antlr4/DmlListener.java
-src/main/java/com/ibm/bi/dml/parser/antlr4/DmlParser.java
-src/main/java/com/ibm/bi/dml/parser/python/PydmlBaseListener.java
-src/main/java/com/ibm/bi/dml/parser/python/PydmlLexer.java
-src/main/java/com/ibm/bi/dml/parser/python/PydmlListener.java
-src/main/java/com/ibm/bi/dml/parser/python/PydmlParser.java
+src/main/java/org/apache/sysml/parser/antlr4/DmlBaseListener.java
+src/main/java/org/apache/sysml/parser/antlr4/DmlLexer.java
+src/main/java/org/apache/sysml/parser/antlr4/DmlListener.java
+src/main/java/org/apache/sysml/parser/antlr4/DmlParser.java
+src/main/java/org/apache/sysml/parser/python/PydmlBaseListener.java
+src/main/java/org/apache/sysml/parser/python/PydmlLexer.java
+src/main/java/org/apache/sysml/parser/python/PydmlListener.java
+src/main/java/org/apache/sysml/parser/python/PydmlParser.java
src/test/scripts/**/in
src/test/scripts/**/out
src/test/scripts/**/expected
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/bin/systemml
----------------------------------------------------------------------
diff --git a/bin/systemml b/bin/systemml
index 08a440d..055e742 100755
--- a/bin/systemml
+++ b/bin/systemml
@@ -55,7 +55,7 @@ USER_DIR=$PWD
BUILD_DIR=${PROJECT_ROOT_DIR}/target
HADOOP_LIB_DIR=${BUILD_DIR}/lib
-DML_SCRIPT_CLASS=${BUILD_DIR}/classes/com/ibm/bi/dml/api/DMLScript.class
+DML_SCRIPT_CLASS=${BUILD_DIR}/classes/org/apache/sysml/api/DMLScript.class
BUILD_ERR_MSG="You must build the project before running this script."
BUILD_DIR_ERR_MSG="Could not find target directory \"${BUILD_DIR}\". ${BUILD_ERR_MSG}"
@@ -130,7 +130,7 @@ CMD="java -Xmx8g -Xms4g -Xmn1g \
-cp $CLASSPATH \
-Dlog4j.configuration=file:'$PROJECT_ROOT_DIR/conf/log4j.properties' \
-Duser.dir='$USER_DIR' \
- com.ibm.bi.dml.api.DMLScript \
+ org.apache.sysml.api.DMLScript \
-f '$SCRIPT_FILE' \
-exec singlenode \
-config='$PROJECT_ROOT_DIR/conf/SystemML-config.xml' \
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/bin/systemml.bat
----------------------------------------------------------------------
diff --git a/bin/systemml.bat b/bin/systemml.bat
index b039a94..f4d593b 100755
--- a/bin/systemml.bat
+++ b/bin/systemml.bat
@@ -111,7 +111,7 @@ set CMD=java -Xmx4g -Xms2g -Xmn400m ^
-cp "%CLASSPATH%" ^
-Dlog4j.configuration=file:"%PROJECT_ROOT_DIR%\conf\log4j.properties" ^
-Duser.dir="%USER_DIR%" ^
- com.ibm.bi.dml.api.DMLScript ^
+ org.apache.sysml.api.DMLScript ^
-f %SCRIPT_FILE% ^
-exec singlenode ^
-config="%PROJECT_ROOT_DIR%\conf\SystemML-config.xml" ^
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/docs/Language Reference/SystemML_Language_Reference.html
----------------------------------------------------------------------
diff --git a/docs/Language Reference/SystemML_Language_Reference.html b/docs/Language Reference/SystemML_Language_Reference.html
index ce3635d..b442da3 100644
--- a/docs/Language Reference/SystemML_Language_Reference.html
+++ b/docs/Language Reference/SystemML_Language_Reference.html
@@ -3673,7 +3673,7 @@ class=SpellE>eval</span>) <o:p></o:p></span></p>
margin-left:.5in;margin-bottom:.0001pt'><span class=GramE><span
style='font-size:11.0pt;font-family:"Courier New";mso-bidi-font-style:italic'>implemented</span></span><span
style='font-size:11.0pt;font-family:"Courier New";mso-bidi-font-style:italic'>
-in (<span class=SpellE>classname</span>="<span class=SpellE>com.ibm.bi.dml.packagesupport.JLapackEigenWrapper</span>")<o:p></o:p></span></p>
+in (<span class=SpellE>classname</span>="<span class=SpellE>org.apache.sysml.packagesupport.JLapackEigenWrapper</span>")<o:p></o:p></span></p>
<p class=MsoNormal>A UDF invocation specifies the function identifier, variable
identifiers for calling parameters, and the variables to be populated by the
@@ -9479,7 +9479,7 @@ shell is as follows:</p>
<p class=MsoNormal style='margin:0in;margin-bottom:.0001pt'><span class=SpellE><span
class=GramE><span style='font-size:11.0pt;font-family:"Courier New";mso-bidi-font-style:
italic'>scala</span></span></span><span style='font-size:11.0pt;font-family:
-"Courier New";mso-bidi-font-style:italic'>> import <span class=SpellE>com.ibm.bi.dml.api.MLContext</span><o:p></o:p></span></p>
+"Courier New";mso-bidi-font-style:italic'>> import <span class=SpellE>org.apache.sysml.api.MLContext</span><o:p></o:p></span></p>
<p class=MsoNormal style='margin:0in;margin-bottom:.0001pt'><o:p> </o:p></p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/docs/dml-language-reference.md
----------------------------------------------------------------------
diff --git a/docs/dml-language-reference.md b/docs/dml-language-reference.md
index dff0886..abebe1f 100644
--- a/docs/dml-language-reference.md
+++ b/docs/dml-language-reference.md
@@ -407,7 +407,7 @@ userParam=value | User-defined parameter to invoke the package. | Yes | Any non-
# example of an external UDF
eigen = externalFunction(matrix[double] A)
return (matrix[double] evec, matrix[double] eval)
- implemented in (classname="com.ibm.bi.dml.packagesupport.JLapackEigenWrapper")
+ implemented in (classname="org.apache.sysml.packagesupport.JLapackEigenWrapper")
A UDF invocation specifies the function identifier, variable identifiers for calling parameters, and the variables to be populated by the returned values from the function. The syntax for function calls is as follows.
@@ -1186,7 +1186,7 @@ The MLContext API allows users to pass RDDs as input/output to SystemML through
Typical usage for MLContext using Spark's Scala Shell is as follows:
- scala> import com.ibm.bi.dml.api.MLContext
+ scala> import org.apache.sysml.api.MLContext
Create input DataFrame from CSV file and potentially perform some feature transformation
[41/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/LeftIndexingOp.java b/src/main/java/com/ibm/bi/dml/hops/LeftIndexingOp.java
deleted file mode 100644
index d22cf67..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/LeftIndexingOp.java
+++ /dev/null
@@ -1,503 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.LeftIndex;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.RangeBasedReIndex;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.ZeroOut;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.UnaryCP.OperationTypes;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-public class LeftIndexingOp extends Hop
-{
- public static LeftIndexingMethod FORCED_LEFT_INDEXING = null;
-
- public enum LeftIndexingMethod {
- SP_GLEFTINDEX, // general case
- SP_MLEFTINDEX //map-only left index where we broadcast right hand side matrix
- }
-
- public static String OPSTRING = "lix"; //"LeftIndexing";
-
- private boolean _rowLowerEqualsUpper = false;
- private boolean _colLowerEqualsUpper = false;
-
- private LeftIndexingOp() {
- //default constructor for clone
- }
-
- public LeftIndexingOp(String l, DataType dt, ValueType vt, Hop inpMatrixLeft, Hop inpMatrixRight, Hop inpRowL, Hop inpRowU, Hop inpColL, Hop inpColU, boolean passedRowsLEU, boolean passedColsLEU) {
- super(l, dt, vt);
-
- getInput().add(0, inpMatrixLeft);
- getInput().add(1, inpMatrixRight);
- getInput().add(2, inpRowL);
- getInput().add(3, inpRowU);
- getInput().add(4, inpColL);
- getInput().add(5, inpColU);
-
- // create hops if one of them is null
- inpMatrixLeft.getParent().add(this);
- inpMatrixRight.getParent().add(this);
- inpRowL.getParent().add(this);
- inpRowU.getParent().add(this);
- inpColL.getParent().add(this);
- inpColU.getParent().add(this);
-
- // set information whether left indexing operation involves row (n x 1) or column (1 x m) matrix
- setRowLowerEqualsUpper(passedRowsLEU);
- setColLowerEqualsUpper(passedColsLEU);
- }
-
-
- public boolean getRowLowerEqualsUpper(){
- return _rowLowerEqualsUpper;
- }
-
- public boolean getColLowerEqualsUpper() {
- return _colLowerEqualsUpper;
- }
-
- public void setRowLowerEqualsUpper(boolean passed){
- _rowLowerEqualsUpper = passed;
- }
-
- public void setColLowerEqualsUpper(boolean passed) {
- _colLowerEqualsUpper = passed;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- try
- {
- ExecType et = optFindExecType();
-
- if(et == ExecType.MR)
- {
- //the right matrix is reindexed
- Lop top=getInput().get(2).constructLops();
- Lop bottom=getInput().get(3).constructLops();
- Lop left=getInput().get(4).constructLops();
- Lop right=getInput().get(5).constructLops();
- /*
- //need to creat new lops for converting the index ranges
- //original range is (a, b) --> (c, d)
- //newa=2-a, newb=2-b
- Lops two=new Data(null, Data.OperationTypes.READ, null, "2", Expression.DataType.SCALAR, Expression.ValueType.INT, false);
- Lops newTop=new Binary(two, top, HopsOpOp2LopsB.get(Hops.OpOp2.MINUS), Expression.DataType.SCALAR, Expression.ValueType.INT, et);
- Lops newLeft=new Binary(two, left, HopsOpOp2LopsB.get(Hops.OpOp2.MINUS), Expression.DataType.SCALAR, Expression.ValueType.INT, et);
- //newc=leftmatrix.row-a+1, newd=leftmatrix.row
- */
- //right hand matrix
- Lop nrow=new UnaryCP(getInput().get(0).constructLops(),
- OperationTypes.NROW, DataType.SCALAR, ValueType.INT);
- Lop ncol=new UnaryCP(getInput().get(0).constructLops(),
- OperationTypes.NCOL, DataType.SCALAR, ValueType.INT);
-
- Lop rightInput = null;
- if (isRightHandSideScalar()) {
- //insert cast to matrix if necessary (for reuse MR runtime)
- rightInput = new UnaryCP(getInput().get(1).constructLops(),
- OperationTypes.CAST_AS_MATRIX,
- DataType.MATRIX, ValueType.DOUBLE);
- rightInput.getOutputParameters().setDimensions( (long)1, (long)1,
- (long)DMLTranslator.DMLBlockSize,
- (long)DMLTranslator.DMLBlockSize,
- (long)-1);
- }
- else
- rightInput = getInput().get(1).constructLops();
-
-
- RangeBasedReIndex reindex = new RangeBasedReIndex(
- rightInput, top, bottom,
- left, right, nrow, ncol,
- getDataType(), getValueType(), et, true);
-
- reindex.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(reindex);
-
- Group group1 = new Group(
- reindex, Group.OperationTypes.Sort, DataType.MATRIX,
- getValueType());
- group1.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(group1);
-
- //the left matrix is zeroed out
- ZeroOut zeroout = new ZeroOut(
- getInput().get(0).constructLops(), top, bottom,
- left, right, getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getDataType(), getValueType(), et);
- zeroout.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(zeroout);
-
- Group group2 = new Group(
- zeroout, Group.OperationTypes.Sort, DataType.MATRIX,
- getValueType());
- group2.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(group2);
-
- Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(Hop.OpOp2.PLUS),
- getDataType(), getValueType(), et);
- binary.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(),
- getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(binary);
-
- setLops(binary);
- }
- else if(et == ExecType.SPARK)
- {
- Hop left = getInput().get(0);
- Hop right = getInput().get(1);
-
- LeftIndexingMethod method = getOptMethodLeftIndexingMethod( right.getDim1(), right.getDim2(),
- right.getRowsInBlock(), right.getColsInBlock(), right.getNnz(), getDataType()==DataType.SCALAR );
- boolean isBroadcast = (method == LeftIndexingMethod.SP_MLEFTINDEX);
-
- //insert cast to matrix if necessary (for reuse broadcast runtime)
- Lop rightInput = right.constructLops();
- if (isRightHandSideScalar()) {
- rightInput = new UnaryCP(rightInput, OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE);
- long bsize = (long)DMLTranslator.DMLBlockSize;
- rightInput.getOutputParameters().setDimensions( 1, 1, bsize, bsize, -1);
- }
-
- LeftIndex leftIndexLop = new LeftIndex(
- left.constructLops(), rightInput,
- getInput().get(2).constructLops(), getInput().get(3).constructLops(),
- getInput().get(4).constructLops(), getInput().get(5).constructLops(),
- getDataType(), getValueType(), et, isBroadcast);
-
- setOutputDimensions(leftIndexLop);
- setLineNumbers(leftIndexLop);
- setLops(leftIndexLop);
- }
- else
- {
- LeftIndex left = new LeftIndex(
- getInput().get(0).constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
- getInput().get(3).constructLops(), getInput().get(4).constructLops(), getInput().get(5).constructLops(),
- getDataType(), getValueType(), et);
-
- setOutputDimensions(left);
- setLineNumbers(left);
- setLops(left);
- }
- }
- catch (Exception e) {
- throw new HopsException(this.printErrorLocation() + "In LeftIndexingOp Hop, error in constructing Lops " , e);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- /**
- * @return true if the right hand side of the indexing operation is a
- * literal.
- */
- private boolean isRightHandSideScalar() {
- Hop rightHandSide = getInput().get(1);
- return (rightHandSide.getDataType() == DataType.SCALAR);
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += OPSTRING;
- return s;
- }
-
- public void printMe() throws HopsException {
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- for (Hop h : getInput()) {
- h.printMe();
- }
- ;
- }
- setVisited(VisitStatus.DONE);
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return false;
- }
-
- @Override
- public void computeMemEstimate( MemoTable memo )
- {
- //overwrites default hops behavior
- super.computeMemEstimate(memo);
-
- //changed final estimate (infer and use input size)
- Hop rhM = getInput().get(1);
- MatrixCharacteristics mcRhM = memo.getAllInputStats(rhM);
- //TODO also use worstcase estimate for output
- if( dimsKnown() && !(rhM.dimsKnown()||mcRhM.dimsKnown()) )
- {
- // unless second input is single cell / row vector / column vector
- // use worst-case memory estimate for second input (it cannot be larger than overall matrix)
- double subSize = -1;
- if( _rowLowerEqualsUpper && _colLowerEqualsUpper )
- subSize = OptimizerUtils.estimateSize(1, 1);
- else if( _rowLowerEqualsUpper )
- subSize = OptimizerUtils.estimateSize(1, _dim2);
- else if( _colLowerEqualsUpper )
- subSize = OptimizerUtils.estimateSize(_dim1, 1);
- else
- subSize = _outputMemEstimate; //worstcase
-
- _memEstimate = getInputSize(0) //original matrix (left)
- + subSize // new submatrix (right)
- + _outputMemEstimate; //output size (output)
- }
- else if ( dimsKnown() && _nnz<0 &&
- _memEstimate>=OptimizerUtils.DEFAULT_SIZE)
- {
- //try a last attempt to infer a reasonable estimate wrt output sparsity
- //(this is important for indexing sparse matrices into empty matrices).
- MatrixCharacteristics mcM1 = memo.getAllInputStats(getInput().get(0));
- MatrixCharacteristics mcM2 = memo.getAllInputStats(getInput().get(1));
- if( mcM1.getNonZeros()>=0 && mcM2.getNonZeros()>=0 ) {
- long lnnz = mcM1.getNonZeros() + mcM2.getNonZeros();
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- _memEstimate = getInputSize(0) //original matrix (left)
- + getInputSize(1) // new submatrix (right)
- + _outputMemEstimate; //output size (output)
- }
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double sparsity = 1.0;
- if( nnz < 0 ) //check for exactly known nnz
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- if( input1.dimsKnown() ) {
- sparsity = OptimizerUtils.getLeftIndexingSparsity(
- input1.getDim1(), input1.getDim2(), input1.getNnz(),
- input2.getDim1(), input2.getDim2(), input2.getNnz());
- }
- }
- else
- {
- sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- }
-
- // The dimensions of the left indexing output is same as that of the first input i.e., getInput().get(0)
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- Hop input1 = getInput().get(0); //original matrix
- Hop input2 = getInput().get(1); //right matrix
- MatrixCharacteristics mc1 = memo.getAllInputStats(input1);
- MatrixCharacteristics mc2 = memo.getAllInputStats(input2);
-
- if( mc1.dimsKnown() ) {
- double sparsity = OptimizerUtils.getLeftIndexingSparsity(
- mc1.getRows(), mc1.getCols(), mc1.getNonZeros(),
- mc2.getRows(), mc2.getCols(), mc2.getNonZeros());
- long lnnz = (long)(sparsity * mc1.getRows() * mc1.getCols());
-
- ret = new long[]{mc1.getRows(), mc1.getCols(), lnnz};
- }
-
- return ret;
- }
-
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- checkAndModifyRecompilationStatus();
- }
- else if ( getInput().get(0).areDimsBelowThreshold() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- return _etype;
- }
-
-
- /**
- *
- * @param m1_dim1
- * @param m1_dim2
- * @param m1_rpb
- * @param m1_cpb
- * @param m2_dim1
- * @param m2_dim2
- * @return
- */
- private LeftIndexingMethod getOptMethodLeftIndexingMethod( long m2_dim1, long m2_dim2,
- long m2_rpb, long m2_cpb, long m2_nnz, boolean isScalar)
- {
- if(FORCED_LEFT_INDEXING != null) {
- return FORCED_LEFT_INDEXING;
- }
-
- // broadcast-based left indexing has memory constraints but is more efficient
- // since it does not require shuffle
- if( isScalar || m2_dim1 >= 1 && m2_dim2 >= 1 // rhs dims known
- && OptimizerUtils.checkSparkBroadcastMemoryBudget(m2_dim1, m2_dim2, m2_rpb, m2_cpb, m2_nnz) )
- {
- return LeftIndexingMethod.SP_MLEFTINDEX;
- }
-
- // default general case
- return LeftIndexingMethod.SP_GLEFTINDEX;
- }
-
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = getInput().get(0); //original matrix
- Hop input2 = getInput().get(1); //rhs matrix
-
- //refresh output dimensions based on original matrix
- setDim1( input1.getDim1() );
- setDim2( input1.getDim2() );
-
- //refresh output nnz if exactly known; otherwise later inference
- if( input1.getNnz() == 0 ) {
- if( input2.getDataType()==DataType.SCALAR )
- setNnz(1);
- else
- setNnz(input2.getNnz());
- }
- else
- setNnz(-1);
- }
-
- /**
- *
- */
- private void checkAndModifyRecompilationStatus()
- {
- // disable recompile for LIX and second input matrix (under certain conditions)
- // if worst-case estimate (2 * original matrix size) was enough to already send it to CP
-
- if( _etype == ExecType.CP )
- {
- _requiresRecompile = false;
-
- Hop rInput = getInput().get(1);
- if( (!rInput.dimsKnown()) && rInput instanceof DataOp )
- {
- //disable recompile for this dataop (we cannot set requiresRecompile directly
- //because we use a top-down traversal for creating lops, hence it would be overwritten)
-
- ((DataOp)rInput).disableRecompileRead();
- }
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- LeftIndexingOp ret = new LeftIndexingOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof LeftIndexingOp)
- || getInput().size() != that.getInput().size() )
- {
- return false;
- }
-
- return ( getInput().get(0) == that.getInput().get(0)
- && getInput().get(1) == that.getInput().get(1)
- && getInput().get(2) == that.getInput().get(2)
- && getInput().get(3) == that.getInput().get(3)
- && getInput().get(4) == that.getInput().get(4)
- && getInput().get(5) == that.getInput().get(5));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/LiteralOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/LiteralOp.java b/src/main/java/com/ibm/bi/dml/hops/LiteralOp.java
deleted file mode 100644
index 0ed419d..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/LiteralOp.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-public class LiteralOp extends Hop
-{
-
- private double value_double = Double.NaN;
- private long value_long = Long.MAX_VALUE;
- private String value_string;
- private boolean value_boolean;
-
- // INT, DOUBLE, STRING, BOOLEAN
-
- private LiteralOp() {
- //default constructor for clone
- }
-
- public LiteralOp(double value) {
- super(String.valueOf(value), DataType.SCALAR, ValueType.DOUBLE);
- this.value_double = value;
- }
-
- public LiteralOp(long value) {
- super(String.valueOf(value), DataType.SCALAR, ValueType.INT);
- this.value_long = value;
- }
-
- public LiteralOp(String value) {
- super(value, DataType.SCALAR, ValueType.STRING);
- this.value_string = value;
- }
-
- public LiteralOp(boolean value) {
- super(String.valueOf(value), DataType.SCALAR, ValueType.BOOLEAN);
- this.value_boolean = value;
- }
-
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
-
- try
- {
- Lop l = null;
-
- switch (getValueType()) {
- case DOUBLE:
- l = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(value_double));
- break;
- case BOOLEAN:
- l = Data.createLiteralLop(ValueType.BOOLEAN, Boolean.toString(value_boolean));
- break;
- case STRING:
- l = Data.createLiteralLop(ValueType.STRING, value_string);
- break;
- case INT:
- l = Data.createLiteralLop(ValueType.INT, Long.toString(value_long));
- break;
- default:
- throw new HopsException(this.printErrorLocation() +
- "unexpected value type constructing lops for LiteralOp.\n");
- }
-
- l.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(l);
- setLops(l);
- }
- catch(LopsException e) {
- throw new HopsException(e);
- }
-
- //note: no reblock lop because always scalar
-
- return getLops();
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- switch (getValueType()) {
- case DOUBLE:
- LOG.debug(" Value: " + value_double);
- break;
- case BOOLEAN:
- LOG.debug(" Value: " + value_boolean);
- break;
- case STRING:
- LOG.debug(" Value: " + value_string);
- break;
- case INT:
- LOG.debug(" Value: " + value_long);
- break;
- default:
- throw new HopsException(this.printErrorLocation() +
- "unexpected value type printing LiteralOp.\n");
- }
-
- for (Hop h : getInput()) {
- h.printMe();
- }
-
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public String getOpString() {
- String val = null;
- switch (getValueType()) {
- case DOUBLE:
- val = Double.toString(value_double);
- break;
- case BOOLEAN:
- val = Boolean.toString(value_boolean);
- break;
- case STRING:
- val = value_string;
- break;
- case INT:
- val = Long.toString(value_long);
- break;
- default:
- val = "";
- }
- return "LiteralOp " + val;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- switch( getValueType() ) {
- case INT:
- ret = OptimizerUtils.INT_SIZE; break;
- case DOUBLE:
- ret = OptimizerUtils.DOUBLE_SIZE; break;
- case BOOLEAN:
- ret = OptimizerUtils.BOOLEAN_SIZE; break;
- case STRING:
- ret = this.value_string.length() * OptimizerUtils.CHAR_SIZE; break;
- case OBJECT:
- ret = OptimizerUtils.DEFAULT_SIZE; break;
- default:
- ret = 0;
- }
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- return null;
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return false;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
- // Since a Literal hop does not represent any computation,
- // this function is not applicable.
- return null;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- //do nothing; it is a scalar
- }
-
- public long getLongValue() throws HopsException
- {
- switch( getValueType() ) {
- case INT:
- return value_long;
- case DOUBLE:
- return UtilFunctions.toLong(value_double);
- case STRING:
- return Long.parseLong(value_string);
- default:
- throw new HopsException("Can not coerce an object of type " + getValueType() + " into Long.");
- }
- }
-
- public double getDoubleValue() throws HopsException {
- switch( getValueType() ) {
- case INT:
- return value_long;
- case DOUBLE:
- return value_double;
- case STRING:
- return Double.parseDouble(value_string);
- default:
- throw new HopsException("Can not coerce an object of type " + getValueType() + " into Double.");
- }
- }
-
- public boolean getBooleanValue() throws HopsException {
- if ( getValueType() == ValueType.BOOLEAN ) {
- return value_boolean;
- }
- else
- throw new HopsException("Can not coerce an object of type " + getValueType() + " into Boolean.");
- }
-
- public String getStringValue()
- {
- switch( getValueType() ) {
- case BOOLEAN:
- return String.valueOf(value_boolean);
- case INT:
- return String.valueOf(value_long);
- case DOUBLE:
- return String.valueOf(value_double);
- case STRING:
- return value_string;
- case OBJECT:
- case UNKNOWN:
- //do nothing (return null)
- }
-
- return null;
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- LiteralOp ret = new LiteralOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret.value_double = value_double;
- ret.value_long = value_long;
- ret.value_string = value_string;
- ret.value_boolean = value_boolean;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/MemoTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/MemoTable.java b/src/main/java/com/ibm/bi/dml/hops/MemoTable.java
deleted file mode 100644
index 428d3e9..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/MemoTable.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.recompile.RecompileStatus;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-/**
- * Memoization Table (hop id, worst-case matrix characteristics).
- *
- */
-public class MemoTable
-{
-
- private HashMap<Long, MatrixCharacteristics> _memo = null;
-
- public MemoTable()
- {
- _memo = new HashMap<Long, MatrixCharacteristics>();
- }
-
- /**
- *
- * @param hops
- * @param status
- */
- public void init( ArrayList<Hop> hops, RecompileStatus status)
- {
- //check existing status
- if( hops == null || hops.isEmpty() || status == null
- || status.getTWriteStats().isEmpty() )
- {
- return; //nothing to do
- }
-
- //population via recursive search for treads
- Hop.resetVisitStatus(hops);
- for( Hop hop : hops )
- rinit(hop, status);
- }
-
- /**
- *
- * @param hop
- * @param status
- */
- public void init( Hop hop, RecompileStatus status)
- {
- //check existing status
- if( hop == null || status == null
- || status.getTWriteStats().isEmpty() )
- {
- return; //nothing to do
- }
-
- //population via recursive search for treads
- hop.resetVisitStatus();
- rinit(hop, status);
- }
-
- /**
- *
- * @param hops
- * @param status
- */
- public void extract( ArrayList<Hop> hops, RecompileStatus status)
- {
- //check existing status
- if( status == null )
- return; //nothing to do
-
- //extract all transient writes (must be dag root)
- for( Hop hop : hops ) {
- if( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTWRITE )
- {
- String varname = hop.getName();
- Hop input = hop.getInput().get(0); //child
- MatrixCharacteristics mc = getAllInputStats(input);
- if( mc != null )
- status.getTWriteStats().put(varname, mc);
- else
- status.getTWriteStats().remove(varname);
- }
- }
- }
-
- /**
- *
- * @param hopID
- * @param dim1
- * @param dim2
- * @param nnz
- */
- public void memoizeStatistics( long hopID, long dim1, long dim2, long nnz )
- {
- _memo.put(hopID, new MatrixCharacteristics(dim1, dim2, -1, -1, nnz));
- }
-
- /**
- *
- * @param inputs
- * @return
- */
- public MatrixCharacteristics[] getAllInputStats( ArrayList<Hop> inputs )
- {
- MatrixCharacteristics[] ret = new MatrixCharacteristics[inputs.size()];
- for( int i=0; i<inputs.size(); i++ )
- {
- Hop input = inputs.get(i);
-
- long dim1 = input.getDim1();
- long dim2 = input.getDim2();
- long nnz = input.getNnz();
-
- if( input.dimsKnown() ) //all dims known
- {
- ret[i] = new MatrixCharacteristics(dim1, dim2, -1, -1, nnz);
- }
- else
- {
- MatrixCharacteristics tmp = _memo.get(input.getHopID());
- if( tmp != null )
- {
- //enrich exact information with worst-case stats
- dim1 = (dim1<=0) ? tmp.getRows() : dim1;
- dim2 = (dim2<=0) ? tmp.getCols() : dim2;
- nnz = (nnz<=0) ? tmp.getNonZeros() : nnz;
- }
- ret[i] = new MatrixCharacteristics(dim1, dim2, -1, -1, nnz);
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param input
- * @return
- */
- public MatrixCharacteristics getAllInputStats( Hop input )
- {
- MatrixCharacteristics ret = null;
-
- long dim1 = input.getDim1();
- long dim2 = input.getDim2();
- long nnz = input.getNnz();
-
- if( input.dimsKnown(true) ) //all dims known
- {
- ret = new MatrixCharacteristics(dim1, dim2, -1, -1, nnz);
- }
- else //enrich exact information with worst-case stats
- {
- MatrixCharacteristics tmp = _memo.get(input.getHopID());
- if( tmp != null ) {
- dim1 = (dim1<=0) ? tmp.getRows() : dim1;
- dim2 = (dim2<=0) ? tmp.getCols() : dim2;
- nnz = (nnz<0) ? tmp.getNonZeros() : nnz;
- }
- ret = new MatrixCharacteristics(dim1, dim2, -1, -1, nnz);
- }
-
- return ret;
- }
-
- /**
- *
- * @param h
- * @return
- */
- public boolean hasInputStatistics(Hop h)
- {
- boolean ret = false;
-
- //determine if any input has useful exact/worst-case stats
- for( Hop in : h.getInput() )
- if( in.dimsKnownAny() || _memo.containsKey(in.getHopID()) ) {
- ret = true;
- break;
- }
-
- //determine if hop itself has worst-case stats (this is important
- //for transient read with cross-dag worst-case estimates)
- if( (h instanceof DataOp && ((DataOp)h).getDataOpType()==DataOpTypes.TRANSIENTREAD)
- ||(h instanceof DataGenOp) )
- {
- ret = true;
- }
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @param status
- */
- private void rinit(Hop hop, RecompileStatus status)
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //probe status of previous twrites
- if( hop instanceof DataOp && hop.getDataType() == DataType.MATRIX
- && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTREAD )
- {
- String varname = hop.getName();
- MatrixCharacteristics mc = status.getTWriteStats().get(varname);
- if( mc != null )
- _memo.put(hop.getHopID(), mc);
- }
-
- if( hop.getInput()!=null && !hop.getInput().isEmpty() )
- for( Hop c : hop.getInput() )
- rinit( c, status );
-
- hop.setVisited(VisitStatus.DONE);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/OptimizerUtils.java b/src/main/java/com/ibm/bi/dml/hops/OptimizerUtils.java
deleted file mode 100644
index 136abd0..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/OptimizerUtils.java
+++ /dev/null
@@ -1,1407 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
-
-public class OptimizerUtils
-{
- private static final Log LOG = LogFactory.getLog(OptimizerUtils.class.getName());
-
- ////////////////////////////////////////////////////////
- // Optimizer constants and flags (incl tuning knobs) //
- ////////////////////////////////////////////////////////
- /**
- * Utilization factor used in deciding whether an operation to be scheduled on CP or MR.
- * NOTE: it is important that MEM_UTIL_FACTOR+CacheableData.CACHING_BUFFER_SIZE < 1.0
- */
- public static double MEM_UTIL_FACTOR = 0.7d;
-
- /**
- * Default memory size, which is used the actual estimate can not be computed
- * -- for example, when input/output dimensions are unknown. In case of ROBUST,
- * the default is set to a large value so that operations are scheduled on MR.
- */
- public static double DEFAULT_SIZE;
-
- public static final long DOUBLE_SIZE = 8;
- public static final long INT_SIZE = 4;
- public static final long CHAR_SIZE = 1;
- public static final long BOOLEAN_SIZE = 1;
- public static final double BIT_SIZE = (double)1/8;
- public static final double INVALID_SIZE = -1d; // memory estimate not computed
-
- public static final long MAX_NUMCELLS_CP_DENSE = Integer.MAX_VALUE;
-
- /**
- * Enables/disables dynamic re-compilation of lops/instructions.
- * If enabled, we recompile each program block that contains at least
- * one hop that requires re-compilation (e.g., unknown statistics
- * during compilation, or program blocks in functions).
- */
- public static boolean ALLOW_DYN_RECOMPILATION = true;
- public static boolean ALLOW_PARALLEL_DYN_RECOMPILATION = ALLOW_DYN_RECOMPILATION && true;
-
- /**
- * Enables/disables to put operations with data-dependent output
- * size into individual statement blocks / program blocks.
- * Since recompilation is done on the granularity of program blocks
- * this enables recompilation of subsequent operations according
- * to the actual output size. This rewrite might limit the opportunity
- * for piggybacking and therefore should only be applied if
- * dyanmic recompilation is enabled as well.
- */
- public static boolean ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS = ALLOW_DYN_RECOMPILATION && true;
-
- /**
- * Enables common subexpression elimination in dags. There is however, a potential tradeoff
- * between computation redundancy and data transfer between MR jobs. Since, we do not reason
- * about transferred data yet, this rewrite rule is enabled by default.
- */
- public static boolean ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = true;
-
- /**
- * Enables constant folding in dags. Constant folding computes simple expressions of binary
- * operations and literals and replaces the hop sub-DAG with a new literal operator.
- */
- public static boolean ALLOW_CONSTANT_FOLDING = true;
-
- /**
- *
- */
- public static boolean ALLOW_ALGEBRAIC_SIMPLIFICATION = true;
-
- /**
- * Enables if-else branch removal for constant predicates (original literals or
- * results of constant folding).
- *
- */
- public static boolean ALLOW_BRANCH_REMOVAL = true;
-
- /**
- *
- */
- public static boolean ALLOW_AUTO_VECTORIZATION = true;
-
- /**
- * Enables simple expression evaluation for datagen parameters 'rows', 'cols'. Simple
- * expressions are defined as binary operations on literals and nrow/ncol. This applies
- * only to exact size information.
- */
- public static boolean ALLOW_SIZE_EXPRESSION_EVALUATION = true;
-
- /**
- * Enables simple expression evaluation for datagen parameters 'rows', 'cols'. Simple
- * expressions are defined as binary operations on literals and b(+) or b(*) on nrow/ncol.
- * This applies also to worst-case size information.
- */
- public static boolean ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION = true;
-
- /**
- *
- */
- public static boolean ALLOW_RAND_JOB_RECOMPILE = true;
-
- /**
- * Enables CP-side data transformation for small files.
- */
- public static boolean ALLOW_TRANSFORM_RECOMPILE = true;
-
- /**
- * Enables parfor runtime piggybacking of MR jobs into the packed jobs for
- * scan sharing.
- */
- public static boolean ALLOW_RUNTIME_PIGGYBACKING = true;
-
- /**
- * Enables interprocedural analysis between main script and functions as well as functions
- * and other functions. This includes, for example, to propagate statistics into functions
- * if save to do so (e.g., if called once).
- */
- public static boolean ALLOW_INTER_PROCEDURAL_ANALYSIS = true;
-
- /**
- * Enables sum product rewrites such as mapmultchains. In the future, this will cover
- * all sum-product related rewrites.
- */
- public static boolean ALLOW_SUM_PRODUCT_REWRITES = true;
-
- /**
- * Enables a specific hop dag rewrite that splits hop dags after csv persistent reads with
- * unknown size in order to allow for recompile.
- */
- public static boolean ALLOW_SPLIT_HOP_DAGS = true;
-
-
- /**
- * Enables parallel read/write of all text formats (textcell, csv, mm)
- * and binary formats (binary block).
- *
- */
- public static boolean PARALLEL_CP_READ_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_READ_BINARYFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_BINARYFORMATS = true;
-
-
-
- /**
- * Specifies a multiplier computing the degree of parallelism of parallel
- * text read/write out of the available degree of parallelism. Set it to 1.0
- * to get a number of threads equal the number of virtual cores.
- *
- */
- public static final double PARALLEL_CP_READ_PARALLELISM_MULTIPLIER = 1.0;
- public static final double PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER = 1.0;
-
- /**
- * Enables multi-threaded matrix multiply for mm, mmchain, and tsmm.
- *
- */
- public static boolean PARALLEL_CP_MATRIX_MULTIPLY = true;
-
- /**
- * Enables the use of CombineSequenceFileInputFormat with splitsize = 2x hdfs blocksize,
- * if sort buffer size large enough and parallelism not hurt. This solves to issues:
- * (1) it combines small files (depending on producers), and (2) it reduces task
- * latency of large jobs with many tasks by factor 2.
- *
- */
- public static final boolean ALLOW_COMBINE_FILE_INPUT_FORMAT = true;
-
-
- //////////////////////
- // Optimizer levels //
- //////////////////////
-
- private static OptimizationLevel _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
-
- /**
- * Optimization Types for Compilation
- *
- * O0 STATIC - Decisions for scheduling operations on CP/MR are based on
- * predefined set of rules, which check if the dimensions are below a
- * fixed/static threshold (OLD Method of choosing between CP and MR).
- * The optimization scope is LOCAL, i.e., per statement block.
- * Advanced rewrites like constant folding, common subexpression elimination,
- * or inter procedural analysis are NOT applied.
- *
- * O1 MEMORY_BASED - Every operation is scheduled on CP or MR, solely
- * based on the amount of memory required to perform that operation.
- * It does NOT take the execution time into account.
- * The optimization scope is LOCAL, i.e., per statement block.
- * Advanced rewrites like constant folding, common subexpression elimination,
- * or inter procedural analysis are NOT applied.
- *
- * O2 MEMORY_BASED - Every operation is scheduled on CP or MR, solely
- * based on the amount of memory required to perform that operation.
- * It does NOT take the execution time into account.
- * The optimization scope is LOCAL, i.e., per statement block.
- * All advanced rewrites are applied. This is the default optimization
- * level of SystemML.
- *
- * O3 GLOBAL TIME_MEMORY_BASED - Operation scheduling on CP or MR as well as
- * many other rewrites of data flow properties such as block size, partitioning,
- * replication, vectorization, etc are done with the optimization objective of
- * minimizing execution time under hard memory constraints per operation and
- * execution context. The optimization scope if GLOBAL, i.e., program-wide.
- * All advanced rewrites are applied. This optimization level requires more
- * optimization time but has higher optimization potential.
- *
- * O4 DEBUG MODE - All optimizations, global and local, which interfere with
- * breakpoints are NOT applied. This optimization level is REQUIRED for the
- * compiler running in debug mode.
- */
- public enum OptimizationLevel {
- O0_LOCAL_STATIC,
- O1_LOCAL_MEMORY_MIN,
- O2_LOCAL_MEMORY_DEFAULT,
- O3_LOCAL_RESOURCE_TIME_MEMORY,
- O4_GLOBAL_TIME_MEMORY,
- O5_DEBUG_MODE,
- };
-
- public static OptimizationLevel getOptLevel() {
- return _optLevel;
- }
-
- public static boolean isMemoryBasedOptLevel() {
- return (_optLevel != OptimizationLevel.O0_LOCAL_STATIC);
- }
-
- public static boolean isOptLevel( OptimizationLevel level ){
- return (_optLevel == level);
- }
-
- /**
- *
- * @param optlevel
- * @throws DMLRuntimeException
- */
- public static void setOptimizationLevel( int optlevel )
- throws DMLRuntimeException
- {
- if( optlevel < 0 || optlevel > 5 )
- throw new DMLRuntimeException("Error: invalid optimization level '"+optlevel+"' (valid values: 0-5).");
-
- // This overrides any optimization level that is present in the configuration file.
- // Why ? This simplifies the calling logic: User doesnot have to maintain two config file or worse
- // edit config file everytime he/she is trying to call the debugger.
- if(DMLScript.ENABLE_DEBUG_MODE) {
- optlevel = 5;
- }
-
- switch( optlevel )
- {
- // opt level 0: static dimensionality
- case 0:
- _optLevel = OptimizationLevel.O0_LOCAL_STATIC;
- ALLOW_CONSTANT_FOLDING = false;
- ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
- ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
- ALLOW_AUTO_VECTORIZATION = false;
- ALLOW_INTER_PROCEDURAL_ANALYSIS = false;
- ALLOW_BRANCH_REMOVAL = false;
- ALLOW_SUM_PRODUCT_REWRITES = false;
- break;
- // opt level 1: memory-based (no advanced rewrites)
- case 1:
- _optLevel = OptimizationLevel.O1_LOCAL_MEMORY_MIN;
- ALLOW_CONSTANT_FOLDING = false;
- ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
- ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
- ALLOW_AUTO_VECTORIZATION = false;
- ALLOW_INTER_PROCEDURAL_ANALYSIS = false;
- ALLOW_BRANCH_REMOVAL = false;
- ALLOW_SUM_PRODUCT_REWRITES = false;
- break;
- // opt level 2: memory-based (all advanced rewrites)
- case 2:
- _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
- break;
- // opt level 3: resource optimization, time- and memory-based (2 w/ resource optimizat)
- case 3:
- _optLevel = OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY;
- break;
-
- // opt level 3: global, time- and memory-based (all advanced rewrites)
- case 4:
- _optLevel = OptimizationLevel.O4_GLOBAL_TIME_MEMORY;
- break;
- // opt level 4: debug mode (no interfering rewrites)
- case 5:
- _optLevel = OptimizationLevel.O5_DEBUG_MODE;
- ALLOW_CONSTANT_FOLDING = false;
- ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
- ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
- ALLOW_INTER_PROCEDURAL_ANALYSIS = false;
- ALLOW_BRANCH_REMOVAL = false;
- ALLOW_DYN_RECOMPILATION = false;
- ALLOW_SIZE_EXPRESSION_EVALUATION = false;
- ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION = false;
- ALLOW_RAND_JOB_RECOMPILE = false;
- ALLOW_SUM_PRODUCT_REWRITES = false;
- ALLOW_SPLIT_HOP_DAGS = false;
- break;
- }
- setDefaultSize();
-
- //handle parallel text io (incl awareness of thread contention in <jdk8)
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_TEXTIO)) {
- PARALLEL_CP_READ_TEXTFORMATS = false;
- PARALLEL_CP_WRITE_TEXTFORMATS = false;
- PARALLEL_CP_READ_BINARYFORMATS = false;
- PARALLEL_CP_WRITE_BINARYFORMATS = false;
- }
- else if( InfrastructureAnalyzer.isJavaVersionLessThanJDK8()
- && InfrastructureAnalyzer.getLocalParallelism() > 1 )
- {
- LOG.warn("Auto-disable multi-threaded text read for 'text' and 'csv' due to thread contention on JRE < 1.8"
- + " (java.version="+ System.getProperty("java.version")+").");
-
- //disable parallel text read
- PARALLEL_CP_READ_TEXTFORMATS = false;
- }
-
- //handle parallel matrix mult / rand configuration
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
- PARALLEL_CP_MATRIX_MULTIPLY = false;
- }
- }
-
- /**
- *
- */
- public static void setDefaultSize()
- {
- //we need to set default_size larger than any execution context
- //memory budget, however, it should not produce overflows on sum
- DEFAULT_SIZE = Math.max( InfrastructureAnalyzer.getLocalMaxMemory(),
- Math.max(InfrastructureAnalyzer.getRemoteMaxMemoryMap(),
- InfrastructureAnalyzer.getRemoteMaxMemoryReduce()));
- }
-
- /**
- * Returns memory budget (according to util factor) in bytes
- *
- * @param localOnly specifies if only budget of current JVM or also MR JVMs
- * @return
- */
- public static double getLocalMemBudget()
- {
- double ret = InfrastructureAnalyzer.getLocalMaxMemory();
- return ret * OptimizerUtils.MEM_UTIL_FACTOR;
- }
-
- /**
- *
- * @return
- */
- public static double getRemoteMemBudgetMap()
- {
- return getRemoteMemBudgetMap(false);
- }
-
-
- /**
- *
- * @return
- */
- public static double getRemoteMemBudgetMap(boolean substractSortBuffer)
- {
- double ret = InfrastructureAnalyzer.getRemoteMaxMemoryMap();
- if( substractSortBuffer )
- ret -= InfrastructureAnalyzer.getRemoteMaxMemorySortBuffer();
- return ret * OptimizerUtils.MEM_UTIL_FACTOR;
- }
-
- /**
- *
- * @return
- */
- public static double getRemoteMemBudgetReduce()
- {
- double ret = InfrastructureAnalyzer.getRemoteMaxMemoryReduce();
- return ret * OptimizerUtils.MEM_UTIL_FACTOR;
- }
-
- /**
- *
- * @param size
- * @return
- */
- public static boolean checkSparkBroadcastMemoryBudget( double size )
- {
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- //basic requirement: the broadcast needs to to fit once in the remote broadcast memory
- //and twice into the local memory budget because we have to create a partitioned broadcast
- //memory and hand it over to the spark context as in-memory object
- return ( size < memBudgetExec && 2*size < memBudgetLocal );
- }
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param nnz
- * @return
- */
- public static boolean checkSparkBroadcastMemoryBudget( long rlen, long clen, long brlen, long bclen, long nnz )
- {
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- double sp = getSparsity(rlen, clen, nnz);
- double size = estimateSizeExactSparsity(rlen, clen, sp);
- double sizeP = estimatePartitionedSizeExactSparsity(rlen, clen, brlen, bclen, sp);
-
- //basic requirement: the broadcast needs to to fit once in the remote broadcast memory
- //and twice into the local memory budget because we have to create a partitioned broadcast
- //memory and hand it over to the spark context as in-memory object
- return ( OptimizerUtils.isValidCPDimensions(rlen, clen)
- && sizeP < memBudgetExec && size+sizeP < memBudgetLocal );
- }
-
- /**
- *
- * @param mc
- * @param memPinned
- * @return
- */
- public static boolean checkSparkCollectMemoryBudget( MatrixCharacteristics mc, long memPinned )
- {
- return checkSparkCollectMemoryBudget(
- mc.getRows(),
- mc.getCols(),
- mc.getRowsPerBlock(),
- mc.getColsPerBlock(),
- mc.getNonZeros(), memPinned);
- }
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param nnz
- * @return
- */
- public static boolean checkSparkCollectMemoryBudget( long rlen, long clen, int brlen, int bclen, long nnz, long memPinned )
- {
- //compute size of output matrix and its blocked representation
- double sp = getSparsity(rlen, clen, nnz);
- double memMatrix = estimateSizeExactSparsity(rlen, clen, sp);
- double memPMatrix = estimatePartitionedSizeExactSparsity(rlen, clen, brlen, bclen, sp);
-
- //check if both output matrix and partitioned matrix fit into local mem budget
- return (memPinned + memMatrix + memPMatrix < getLocalMemBudget());
- }
-
- /**
- * Returns the number of reducers that potentially run in parallel.
- * This is either just the configured value (SystemML config) or
- * the minimum of configured value and available reduce slots.
- *
- * @param configOnly
- * @return
- */
- public static int getNumReducers( boolean configOnly )
- {
- int ret = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
- if( !configOnly ) {
- ret = Math.min(ret,InfrastructureAnalyzer.getRemoteParallelReduceTasks());
-
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() )
- ret = (int)Math.max( ret, YarnClusterAnalyzer.getNumCores()/2 );
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public static int getNumMappers()
- {
- int ret = InfrastructureAnalyzer.getRemoteParallelMapTasks();
-
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() )
- ret = (int)Math.max( ret, YarnClusterAnalyzer.getNumCores() );
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public static boolean isSparkExecutionMode() {
- return ( DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK
- || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK);
- }
-
- /**
- *
- * @return
- */
- public static boolean isHybridExecutionMode() {
- return ( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID
- || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK );
- }
-
- /**
- * Returns the degree of parallelism used for parallel text read.
- * This is computed as the number of virtual cores scales by the
- * PARALLEL_READ_PARALLELISM_MULTIPLIER. If PARALLEL_READ_TEXTFORMATS
- * is disabled, this method returns 1.
- *
- * @return
- */
- public static int getParallelTextReadParallelism()
- {
- if( !PARALLEL_CP_READ_TEXTFORMATS )
- return 1; // sequential execution
-
- //compute degree of parallelism for parallel text read
- double dop = InfrastructureAnalyzer.getLocalParallelism()
- * PARALLEL_CP_READ_PARALLELISM_MULTIPLIER;
- return (int) Math.round(dop);
- }
-
- /**
- *
- * @return
- */
- public static int getParallelBinaryReadParallelism()
- {
- if( !PARALLEL_CP_READ_BINARYFORMATS )
- return 1; // sequential execution
-
- //compute degree of parallelism for parallel text read
- double dop = InfrastructureAnalyzer.getLocalParallelism()
- * PARALLEL_CP_READ_PARALLELISM_MULTIPLIER;
- return (int) Math.round(dop);
- }
-
- /**
- * Returns the degree of parallelism used for parallel text write.
- * This is computed as the number of virtual cores scales by the
- * PARALLEL_WRITE_PARALLELISM_MULTIPLIER. If PARALLEL_WRITE_TEXTFORMATS
- * is disabled, this method returns 1.
- *
- * @return
- */
- public static int getParallelTextWriteParallelism()
- {
- if( !PARALLEL_CP_WRITE_TEXTFORMATS )
- return 1; // sequential execution
-
- //compute degree of parallelism for parallel text read
- double dop = InfrastructureAnalyzer.getLocalParallelism()
- * PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER;
- return (int) Math.round(dop);
- }
-
- /**
- *
- * @return
- */
- public static int getParallelBinaryWriteParallelism()
- {
- if( !PARALLEL_CP_WRITE_BINARYFORMATS )
- return 1; // sequential execution
-
- //compute degree of parallelism for parallel text read
- double dop = InfrastructureAnalyzer.getLocalParallelism()
- * PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER;
- return (int) Math.round(dop);
- }
-
- ////////////////////////
- // Memory Estimates //
- ////////////////////////
-
- /**
- *
- * @param mc
- * @return
- */
- public static long estimateSizeExactSparsity(MatrixCharacteristics mc)
- {
- return estimateSizeExactSparsity(
- mc.getRows(),
- mc.getCols(),
- mc.getNonZeros());
- }
-
- /**
- * Estimates the footprint (in bytes) for an in-memory representation of a
- * matrix with dimensions=(nrows,ncols) and and number of non-zeros nnz.
- *
- * @param nrows
- * @param ncols
- * @param sp
- * @return
- */
- public static long estimateSizeExactSparsity(long nrows, long ncols, long nnz)
- {
- double sp = getSparsity(nrows, ncols, nnz);
- return estimateSizeExactSparsity(nrows, ncols, sp);
- }
-
- /**
- * Estimates the footprint (in bytes) for an in-memory representation of a
- * matrix with dimensions=(nrows,ncols) and sparsity=sp.
- *
- * This function can be used directly in Hops, when the actual sparsity is
- * known i.e., <code>sp</code> is guaranteed to give worst-case estimate
- * (e.g., Rand with a fixed sparsity). In all other cases, estimateSize()
- * must be used so that worst-case estimates are computed, whenever
- * applicable.
- *
- * @param nrows
- * @param ncols
- * @param sp
- * @return
- */
- public static long estimateSizeExactSparsity(long nrows, long ncols, double sp)
- {
- return MatrixBlock.estimateSizeInMemory(nrows,ncols,sp);
- }
-
- /**
- * Estimates the footprint (in bytes) for a partitioned in-memory representation of a
- * matrix with the given matrix characteristics
- *
- * @param mc
- * @return
- */
- public static long estimatePartitionedSizeExactSparsity(MatrixCharacteristics mc)
- {
- return estimatePartitionedSizeExactSparsity(
- mc.getRows(),
- mc.getCols(),
- mc.getRowsPerBlock(),
- mc.getColsPerBlock(),
- mc.getNonZeros());
- }
-
- /**
- * Estimates the footprint (in bytes) for a partitioned in-memory representation of a
- * matrix with dimensions=(nrows,ncols) and number of non-zeros nnz.
- *
- * @param nrows
- * @param ncols
- * @param sp
- * @return
- */
- public static long estimatePartitionedSizeExactSparsity(long rlen, long clen, long brlen, long bclen, long nnz)
- {
- double sp = getSparsity(rlen, clen, nnz);
- return estimatePartitionedSizeExactSparsity(rlen, clen, brlen, bclen, sp);
- }
-
- /**
- * Estimates the footprint (in bytes) for a partitioned in-memory representation of a
- * matrix with dimensions=(nrows,ncols) and sparsity=sp.
- *
- * @param nrows
- * @param ncols
- * @param sp
- * @return
- */
- public static long estimatePartitionedSizeExactSparsity(long rlen, long clen, long brlen, long bclen, double sp)
- {
- long ret = 0;
-
- //check for guaranteed existence of empty blocks (less nnz than total number of blocks)
- long tnrblks = (long)Math.ceil((double)rlen/brlen);
- long tncblks = (long)Math.ceil((double)clen/bclen);
- long nnz = (long) Math.ceil(sp * rlen * clen);
- if( nnz < tnrblks * tncblks ) {
- long lrlen = Math.min(rlen, brlen);
- long lclen = Math.min(clen, bclen);
- return nnz * estimateSizeExactSparsity(lrlen, lclen, 1)
- + (tnrblks * tncblks - nnz) * estimateSizeEmptyBlock(lrlen, lclen);
- }
-
- //estimate size of full brlen x bclen blocks
- long nrblks = rlen / brlen;
- long ncblks = clen / bclen;
- if( nrblks * ncblks > 0 )
- ret += nrblks * ncblks * estimateSizeExactSparsity(brlen, bclen, sp);
-
- //estimate size of bottom boundary blocks
- long lrlen = rlen % brlen;
- if( ncblks > 0 && lrlen > 0 )
- ret += ncblks * estimateSizeExactSparsity(lrlen, bclen, sp);
-
- //estimate size of right boundary blocks
- long lclen = clen % bclen;
- if( nrblks > 0 && lclen > 0 )
- ret += nrblks * estimateSizeExactSparsity(brlen, lclen, sp);
-
- //estimate size of bottom right boundary block
- if( lrlen > 0 && lclen > 0 )
- ret += estimateSizeExactSparsity(lrlen, lclen, sp);
-
- return ret;
- }
-
- /**
- * Similar to estimate() except that it provides worst-case estimates
- * when the optimization type is ROBUST.
- *
- * @param nrows
- * @param ncols
- * @param sp
- * @return
- */
- public static long estimateSize(long nrows, long ncols)
- {
- return estimateSizeExactSparsity(nrows, ncols, 1.0);
- }
-
- /**
- *
- * @param nrows
- * @param ncols
- * @return
- */
- public static long estimateSizeEmptyBlock(long nrows, long ncols)
- {
- return estimateSizeExactSparsity(0, 0, 0.0d);
- }
-
- /**
- * Estimates the memory footprint of a SparseRow with <code>clen</code>
- * columns and <code>sp</code> sparsity. This method accounts for the
- * overhead incurred by extra cells allocated (but not used) for SparseRow.
- * It assumes that non-zeros are uniformly distributed in the matrix --
- * i.e., #estimated nnz in a given SparseRow = clen*sp.
- *
- * @param clen
- * @param sp
- * @return estimated size in bytes
- */
- public static long estimateRowSize(long clen, double sp)
- {
- if ( sp == 0 )
- return 0;
-
- int basicSize = 28;
- int cellSize = 12; // every cell takes 12 (8+4) bytes
- if ( sp == 1 ) {
- return clen * cellSize;
- }
- long numCells = SparseRow.initialCapacity;
- if ( (long) (sp*clen) > numCells ) {
- numCells = (long) (sp*clen);
- }
- long allocatedCells = (long)Math.pow(2, Math.ceil(Math.log(numCells)/Math.log(2)) );
- long rowSize = basicSize + allocatedCells * cellSize;
- return rowSize;
- }
-
- public static long estimateSizeTextOutput( long rows, long cols, long nnz, OutputInfo oinfo )
- {
- long bsize = MatrixBlock.estimateSizeOnDisk(rows, cols, nnz);
- if( oinfo == OutputInfo.TextCellOutputInfo || oinfo == OutputInfo.MatrixMarketOutputInfo )
- return bsize * 3;
- else if( oinfo == OutputInfo.CSVOutputInfo )
- return bsize * 2;
-
- //unknown output info
- return bsize;
- }
-
- /**
- * Returns false if dimensions known to be invalid; other true
- *
- * @param rows
- * @param cols
- * @return
- */
- public static boolean isValidCPDimensions( long rows, long cols )
- {
- //the current CP runtime implementation requires that rows and cols
- //are integers since we use a single matrixblock to represent the
- //entire matrix
- return (rows <= Integer.MAX_VALUE && cols<=Integer.MAX_VALUE);
- }
-
- /**
- * Determines if valid matrix size to be represented in CP data structures. Note that
- * sparsity needs to be specified as rows*cols if unknown.
- *
- * @param rows
- * @param cols
- * @param sparsity
- * @return
- */
- public static boolean isValidCPMatrixSize( long rows, long cols, double sparsity )
- {
- boolean ret = true;
-
- //the current CP runtime implementation has several limitations:
- //1) for dense: 16GB because we use a linearized array (bounded to int in java)
- //2) for sparse: 2G x 2G nnz because (1) nnz maintained as long, (2) potential changes
- // to dense, and (3) sparse row arrays also of max int size (worst case in case of skew)
- long nnz = (long)(sparsity * rows * cols);
- boolean sparse = MatrixBlock.evalSparseFormatInMemory(rows, cols, nnz);
-
- if( sparse ) //SPARSE
- {
- //check max nnz
- ret = (nnz <= Long.MAX_VALUE);
- }
- else //DENSE
- {
- //check number of matrix cell
- ret = ((rows * cols) <= MAX_NUMCELLS_CP_DENSE);
- }
-
- return ret;
- }
-
-
- /**
- *
- * @return
- * @throws HopsException
- */
- public static boolean allowsToFilterEmptyBlockOutputs( Hop hop )
- throws HopsException
- {
- boolean ret = true;
- for( Hop p : hop.getParent() ) {
- p.optFindExecType(); //ensure exec type evaluated
- ret &= ( p.getExecType()==ExecType.CP
- ||(p instanceof AggBinaryOp && allowsToFilterEmptyBlockOutputs(p) )
- ||(p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.PERSISTENTWRITE && ((DataOp)p).getInputFormatType()==FileFormatTypes.TEXT))
- && !(p instanceof FunctionOp || (p instanceof DataOp && ((DataOp)p).getInputFormatType()!=FileFormatTypes.TEXT) ); //no function call or transient write
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public static int getConstrainedNumThreads(int maxNumThreads)
- {
- //by default max local parallelism (vcores)
- int ret = InfrastructureAnalyzer.getLocalParallelism();
-
- //apply external max constraint (e.g., set by parfor or other rewrites)
- if( maxNumThreads > 0 ) {
- ret = Math.min(ret, maxNumThreads);
- }
-
- //apply global multi-threading constraint
- if( !PARALLEL_CP_MATRIX_MULTIPLY ) {
- ret = 1;
- }
-
- return ret;
- }
-
- ////////////////////////
- // Sparsity Estimates //
- ////////////////////////
-
- /**
- * Estimates the result sparsity for Matrix Multiplication A %*% B.
- *
- * @param sp1 -- sparsity of A
- * @param sp2 -- sparsity of B
- * @param m -- nrow(A)
- * @param k -- ncol(A), nrow(B)
- * @param n -- ncol(B)
- * @return
- */
- public static double getMatMultSparsity(double sp1, double sp2, long m, long k, long n, boolean worstcase)
- {
- if( worstcase ){
- double nnz1 = sp1 * m * k;
- double nnz2 = sp2 * k * n;
- return Math.min(1, nnz1/m) * Math.min(1, nnz2/n);
- }
- else
- return (1 - Math.pow(1-sp1*sp2, k) );
- }
-
- /**
- *
- * @param rlen1
- * @param clen1
- * @param nnz1
- * @param rlen2
- * @param clen2
- * @param nnz2
- * @return
- */
- public static double getLeftIndexingSparsity( long rlen1, long clen1, long nnz1, long rlen2, long clen2, long nnz2 )
- {
- boolean scalarRhs = (rlen2==0 && clen2==0);
-
- //infer output worstcase output nnz
- long lnnz = -1;
- if( nnz1>=0 && scalarRhs )
- lnnz = nnz1+1; // nnz(left) + scalar
- else if( nnz1>=0 && nnz2>=0 )
- lnnz = nnz1 + nnz2; // nnz(left) + nnz(right)
- else if( nnz1>=0 && rlen2>0 && clen2>0 )
- lnnz = nnz1 + rlen2*clen2; // nnz(left) + nnz(right_dense)
- lnnz = Math.min(lnnz, rlen1*clen1);
-
- return getSparsity(rlen1, clen1, (lnnz>=0) ? lnnz : rlen1*clen1);
- }
-
- /**
- * Determines if a given binary op is potentially conditional sparse safe.
- *
- * @param op
- * @return
- */
- public static boolean isBinaryOpConditionalSparseSafe( OpOp2 op )
- {
- return ( op==OpOp2.GREATER
- || op==OpOp2.LESS
- || op==OpOp2.NOTEQUAL
- || op==OpOp2.EQUAL
- || op==OpOp2.MINUS);
- }
-
- /**
- * Determines if a given binary op with scalar literal guarantee an output
- * sparsity which is exactly the same as its matrix input sparsity.
- *
- * @param op
- * @param lit
- * @return
- */
- public static boolean isBinaryOpConditionalSparseSafeExact( OpOp2 op, LiteralOp lit )
- {
- double val = HopRewriteUtils.getDoubleValueSafe(lit);
-
- return ( op==OpOp2.NOTEQUAL && val==0);
- }
-
- /**
- *
- * @param sp1
- * @param op
- * @param lit
- * @return
- */
- public static double getBinaryOpSparsityConditionalSparseSafe( double sp1, OpOp2 op, LiteralOp lit )
- {
- double val = HopRewriteUtils.getDoubleValueSafe(lit);
-
- return ( (op==OpOp2.GREATER && val==0)
- ||(op==OpOp2.LESS && val==0)
- ||(op==OpOp2.NOTEQUAL && val==0)
- ||(op==OpOp2.EQUAL && val!=0)
- ||(op==OpOp2.MINUS && val==0)) ? sp1 : 1.0;
- }
-
- /**
- * Estimates the result sparsity for matrix-matrix binary operations (A op B)
- *
- * @param sp1 -- sparsity of A
- * @param sp2 -- sparsity of B
- * @param op -- binary operation
- * @return
- *
- * NOTE: append has specific computation
- */
- public static double getBinaryOpSparsity(double sp1, double sp2, OpOp2 op, boolean worstcase)
- {
- // default is worst-case estimate for robustness
- double ret = 1.0;
-
- if( worstcase )
- {
- //NOTE: for matrix-scalar operations this estimate is too conservative, because
- //Math.min(1, sp1 + sp2) will always give a sparsity 1 if we pass sp2=1 for scalars.
- //In order to do better (with guarantees), we need to take the actual values into account
- switch(op)
- {
- case PLUS:
- case MINUS:
- case LESS:
- case GREATER:
- case NOTEQUAL:
- case MIN:
- case MAX:
- case OR:
- ret = Math.min(1, sp1 + sp2); break;
- case MULT:
- case AND:
- ret = Math.min(sp1, sp2); break;
- case DIV:
- case MODULUS:
- case POW:
- case MINUS_NZ:
- case LOG_NZ:
- ret = sp1; break;
- //case EQUAL: //doesnt work on worstcase estimates, but on
- // ret = 1-Math.abs(sp1-sp2); break;
-
- default:
- ret = 1.0;
- }
- }
- else
- {
- switch(op) {
- case PLUS:
- case MINUS:
- // result[i,j] != 0 iff A[i,j] !=0 || B[i,j] != 0
- // worst case estimate = sp1+sp2
- ret = (1 - (1-sp1)*(1-sp2));
- break;
-
- case MULT:
- // result[i,j] != 0 iff A[i,j] !=0 && B[i,j] != 0
- // worst case estimate = min(sp1,sp2)
- ret = sp1 * sp2;
- break;
-
- case DIV:
- ret = 1.0; // worst case estimate
- break;
-
- case LESS:
- case LESSEQUAL:
- case GREATER:
- case GREATEREQUAL:
- case EQUAL:
- case NOTEQUAL:
- ret = 1.0; // purely data-dependent operations, and hence worse-case estimate
- break;
-
- //MIN, MAX, AND, OR, LOG, POW
- default:
- ret = 1.0;
- }
- }
-
- return ret;
- }
-
- public static double getSparsity( long dim1, long dim2, long nnz )
- {
- if( dim1<=0 || dim2<=0 || nnz<0 )
- return 1.0;
- else
- return Math.min(((double)nnz)/dim1/dim2, 1.0);
- }
-
- public static String toMB(double inB) {
- if ( inB < 0 )
- return "-";
- return String.format("%.0f", inB/(1024*1024) );
- }
-
-
-
- /**
- * Function to evaluate simple size expressions over literals and now/ncol.
- *
- * It returns the exact results of this expressions if known, otherwise
- * Long.MAX_VALUE if unknown.
- *
- * @param root
- * @return
- * @throws HopsException
- */
- public static long rEvalSimpleLongExpression( Hop root, HashMap<Long, Long> valMemo )
- throws HopsException
- {
- long ret = Long.MAX_VALUE;
-
- //for simplicity and robustness call double and cast.
- HashMap<Long, Double> dvalMemo = new HashMap<Long, Double>();
- double tmp = rEvalSimpleDoubleExpression(root, dvalMemo);
- if( tmp!=Double.MAX_VALUE )
- ret = UtilFunctions.toLong( tmp );
-
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @param vars
- * @return
- * @throws HopsException
- */
- public static long rEvalSimpleLongExpression( Hop root, HashMap<Long, Long> valMemo, LocalVariableMap vars )
- throws HopsException
- {
- long ret = Long.MAX_VALUE;
-
- //for simplicity and robustness call double and cast.
- HashMap<Long, Double> dvalMemo = new HashMap<Long, Double>();
- double tmp = rEvalSimpleDoubleExpression(root, dvalMemo, vars);
- if( tmp!=Double.MAX_VALUE )
- ret = UtilFunctions.toLong( tmp );
-
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @return
- * @throws HopsException
- */
- public static double rEvalSimpleDoubleExpression( Hop root, HashMap<Long, Double> valMemo )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- //always use constants
- if( root instanceof LiteralOp )
- ret = HopRewriteUtils.getDoubleValue((LiteralOp)root);
-
- //advanced size expression evaluation
- if( OptimizerUtils.ALLOW_SIZE_EXPRESSION_EVALUATION )
- {
- if( root instanceof UnaryOp )
- ret = rEvalSimpleUnaryDoubleExpression(root, valMemo);
- else if( root instanceof BinaryOp )
- ret = rEvalSimpleBinaryDoubleExpression(root, valMemo);
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @param vars
- * @return
- * @throws HopsException
- */
- public static double rEvalSimpleDoubleExpression( Hop root, HashMap<Long, Double> valMemo, LocalVariableMap vars )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- if( OptimizerUtils.ALLOW_SIZE_EXPRESSION_EVALUATION )
- {
- if( root instanceof LiteralOp )
- ret = HopRewriteUtils.getDoubleValue((LiteralOp)root);
- else if( root instanceof UnaryOp )
- ret = rEvalSimpleUnaryDoubleExpression(root, valMemo, vars);
- else if( root instanceof BinaryOp )
- ret = rEvalSimpleBinaryDoubleExpression(root, valMemo, vars);
- else if( root instanceof DataOp ) {
- String name = root.getName();
- Data dat = vars.get(name);
- if( dat!=null && dat instanceof ScalarObject )
- ret = ((ScalarObject)dat).getDoubleValue();
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
-
- /**
- *
- * @param root
- * @param valMemo
- * @return
- * @throws HopsException
- */
- protected static double rEvalSimpleUnaryDoubleExpression( Hop root, HashMap<Long, Double> valMemo )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- UnaryOp uroot = (UnaryOp) root;
- Hop input = uroot.getInput().get(0);
-
- if(uroot.getOp() == Hop.OpOp1.NROW)
- ret = (input.getDim1()>0) ? input.getDim1() : Double.MAX_VALUE;
- else if( uroot.getOp() == Hop.OpOp1.NCOL )
- ret = (input.getDim2()>0) ? input.getDim2() : Double.MAX_VALUE;
- else
- {
- double lval = rEvalSimpleDoubleExpression(uroot.getInput().get(0), valMemo);
- if( lval != Double.MAX_VALUE )
- {
- switch( uroot.getOp() )
- {
- case SQRT: ret = Math.sqrt(lval); break;
- case ROUND: ret = Math.round(lval); break;
- case CAST_AS_BOOLEAN: ret = (lval!=0)? 1 : 0; break;
- case CAST_AS_INT: ret = UtilFunctions.toLong(lval); break;
- case CAST_AS_DOUBLE: ret = lval; break;
- default: ret = Double.MAX_VALUE;
- }
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @param vars
- * @return
- * @throws HopsException
- */
- protected static double rEvalSimpleUnaryDoubleExpression( Hop root, HashMap<Long, Double> valMemo, LocalVariableMap vars )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- UnaryOp uroot = (UnaryOp) root;
- Hop input = uroot.getInput().get(0);
-
- if(uroot.getOp() == Hop.OpOp1.NROW)
- ret = (input.getDim1()>0) ? input.getDim1() : Double.MAX_VALUE;
- else if( uroot.getOp() == Hop.OpOp1.NCOL )
- ret = (input.getDim2()>0) ? input.getDim2() : Double.MAX_VALUE;
- else
- {
- double lval = rEvalSimpleDoubleExpression(uroot.getInput().get(0), valMemo, vars);
- if( lval != Double.MAX_VALUE )
- {
- switch( uroot.getOp() )
- {
- case SQRT: ret = Math.sqrt(lval); break;
- case ROUND: ret = Math.round(lval); break;
- case CAST_AS_BOOLEAN: ret = (lval!=0)? 1 : 0; break;
- case CAST_AS_INT: ret = UtilFunctions.toLong(lval); break;
- case CAST_AS_DOUBLE: ret = lval; break;
- default: ret = Double.MAX_VALUE;
- }
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @return
- * @throws HopsException
- */
- protected static double rEvalSimpleBinaryDoubleExpression( Hop root, HashMap<Long, Double> valMemo )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- BinaryOp broot = (BinaryOp) root;
-
- double lret = rEvalSimpleDoubleExpression(broot.getInput().get(0), valMemo);
- double rret = rEvalSimpleDoubleExpression(broot.getInput().get(1), valMemo);
- //note: positive and negative values might be valid subexpressions
- if( lret!=Double.MAX_VALUE && rret!=Double.MAX_VALUE ) //if known
- {
- switch( broot.getOp() )
- {
- case PLUS: ret = lret + rret; break;
- case MINUS: ret = lret - rret; break;
- case MULT: ret = lret * rret; break;
- case DIV: ret = lret / rret; break;
- case MIN: ret = Math.min(lret, rret); break;
- case MAX: ret = Math.max(lret, rret); break;
- case POW: ret = Math.pow(lret, rret); break;
- default: ret = Double.MAX_VALUE;
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
- /**
- *
- * @param root
- * @param valMemo
- * @param vars
- * @return
- * @throws HopsException
- */
- protected static double rEvalSimpleBinaryDoubleExpression( Hop root, HashMap<Long, Double> valMemo, LocalVariableMap vars )
- throws HopsException
- {
- //memoization (prevent redundant computation of common subexpr)
- if( valMemo.containsKey(root.getHopID()) )
- return valMemo.get(root.getHopID());
-
- double ret = Double.MAX_VALUE;
-
- BinaryOp broot = (BinaryOp) root;
-
- double lret = rEvalSimpleDoubleExpression(broot.getInput().get(0), valMemo, vars);
- double rret = rEvalSimpleDoubleExpression(broot.getInput().get(1), valMemo, vars);
- //note: positive and negative values might be valid subexpressions
- if( lret!=Double.MAX_VALUE && rret!=Double.MAX_VALUE ) //if known
- {
- switch( broot.getOp() )
- {
- case PLUS: ret = lret + rret; break;
- case MINUS: ret = lret - rret; break;
- case MULT: ret = lret * rret; break;
- case DIV: ret = lret / rret; break;
- case MIN: ret = Math.min(lret, rret); break;
- case MAX: ret = Math.max(lret, rret); break;
- case POW: ret = Math.pow(lret, rret); break;
- default: ret = Double.MAX_VALUE;
- }
- }
-
- valMemo.put(root.getHopID(), ret);
- return ret;
- }
-
-
-}
[57/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarLeftTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarLeftTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarLeftTest.java
index 28b91d4..3827e30 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarLeftTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarLeftTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,12 +23,12 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* The main purpose of this test is to verify the internal optimization regarding
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarRightTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarRightTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarRightTest.java
index 6ccdcde..bef64a8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarRightTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPPredScalarRightTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.util.HashMap;
@@ -23,12 +23,12 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* The main purpose of this test is to verify the internal optimization regarding
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
index c5174e8..7f0e273 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.matrix_full_other;
+package org.apache.sysml.test.integration.functions.binary.matrix_full_other;
import java.io.IOException;
import java.util.HashMap;
@@ -24,17 +24,17 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullPowerTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AdditionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AdditionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AdditionTest.java
index e7c7172..85e809a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AdditionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AdditionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AndTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AndTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AndTest.java
index 6dd3cfe..ee2522d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AndTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/AndTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/DivisionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/DivisionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/DivisionTest.java
index d7bdcb6..c6f13bc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/DivisionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/DivisionTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class DivisionTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/EqualTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/EqualTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/EqualTest.java
index 357c206..7f11a99 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/EqualTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/EqualTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/FullStringComparisonTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/FullStringComparisonTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/FullStringComparisonTest.java
index 8c504e0..2783f6a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/FullStringComparisonTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/FullStringComparisonTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
* The main purpose of this test is to verify all combinations of
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanOrEqualTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanOrEqualTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanOrEqualTest.java
index 78b54a6..42b4cbb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanOrEqualTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanOrEqualTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanTest.java
index d197560..54efaf0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/GreaterThanTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanOrEqualTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanOrEqualTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanOrEqualTest.java
index e2ac678..8fa51d4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanOrEqualTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanOrEqualTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanTest.java
index f63471f..16783c9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LessThanTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LogarithmTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LogarithmTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LogarithmTest.java
index fe25242..b803fb1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LogarithmTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/LogarithmTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/ModulusTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/ModulusTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/ModulusTest.java
index 11bac0e..3561bc8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/ModulusTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/ModulusTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ModulusTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/MultiplicationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/MultiplicationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/MultiplicationTest.java
index ac109b6..0ac0048 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/MultiplicationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/MultiplicationTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/OrTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/OrTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/OrTest.java
index a5f6f46..d302d1e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/OrTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/OrTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/PowerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/PowerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/PowerTest.java
index c1c63b5..4862507 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/PowerTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/PowerTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/SubtractionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/SubtractionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/SubtractionTest.java
index 10d3dcf..02c2da9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/SubtractionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/scalar/SubtractionTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.binary.scalar;
+package org.apache.sysml.test.integration.functions.binary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class SubtractionTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/blocks/VariableTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/blocks/VariableTest.java b/src/test/java/org/apache/sysml/test/integration/functions/blocks/VariableTest.java
index 2deb9c9..cbb49cf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/blocks/VariableTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/blocks/VariableTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.blocks;
+package org.apache.sysml.test.integration.functions.blocks;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/blocks/WhileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/blocks/WhileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/blocks/WhileTest.java
index 1462c4b..d4b4054 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/blocks/WhileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/blocks/WhileTest.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.blocks;
+package org.apache.sysml.test.integration.functions.blocks;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java b/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
index bf3d67b..d75d559 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.caching;
+package org.apache.sysml.test.integration.functions.caching;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class CachingPWriteExportTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/FullReblockTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/FullReblockTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/FullReblockTest.java
index 838854e..e043b1d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/FullReblockTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/FullReblockTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullReblockTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
index 60efd08..4be4fdf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandRuntimePlatformTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandRuntimePlatformTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandRuntimePlatformTest.java
index d1829b9..3ddccd0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandRuntimePlatformTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandRuntimePlatformTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.util.Arrays;
import java.util.Collection;
@@ -27,12 +27,12 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Complete suit of tests for Rand:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest1.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest1.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest1.java
index 7316402..d8a3b94 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest1.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest1.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest2.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest2.java
index 01b2a42..9fcd90d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest2.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest2.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest3.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest3.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest3.java
index 47fd883..1dc0dfb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest3.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest3.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest4.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest4.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest4.java
index 42e8b05..90d907c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest4.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest4.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest5.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest5.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest5.java
index 26b18dc..a8b17c6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest5.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandTest5.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* The major purpose of this test is to verify min/max value domains
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarMinMaxTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarMinMaxTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarMinMaxTest.java
index 671771f..570dccf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarMinMaxTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarMinMaxTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarSeedTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarSeedTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarSeedTest.java
index b9f6fe2..c6147ba 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarSeedTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/RandVarSeedTest.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.util.HashMap;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
index 0afef18..6bf290c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import static org.junit.Assert.fail;
@@ -23,15 +23,15 @@ import java.io.IOException;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/ReblockTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/ReblockTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/ReblockTest.java
index 11a1385..72b5992 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/ReblockTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/ReblockTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/SampleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/SampleTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/SampleTest.java
index bfa7a5c..1a2cd57 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/SampleTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/SampleTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.util.Arrays;
import java.util.Collection;
@@ -25,11 +25,11 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
* Tests if Rand produces the same output, for a given set of parameters, across different (CP vs. MR) runtime platforms.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/SequenceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/SequenceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/SequenceTest.java
index 30f4378..710f740 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/SequenceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/SequenceTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import java.util.Arrays;
import java.util.Collection;
@@ -26,12 +26,12 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Tests if Rand produces the same output, for a given set of parameters, across different (CP vs. MR) runtime platforms.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/VariableTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/VariableTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/VariableTest.java
index 6efa75b..316cb22 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/VariableTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/VariableTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteMMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteMMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteMMTest.java
index 0ddeb80..84daf99 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteMMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteMMTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
index f03d455..0cffd7d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.data;
+package org.apache.sysml.test.integration.functions.data;
import static org.junit.Assert.*;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.BinaryMatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.BinaryMatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java
index 1c6384b..65de627 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.dmlscript;
+package org.apache.sysml.test.integration.functions.dmlscript;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java
index e870667..6aa19fb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.dmlscript;
+package org.apache.sysml.test.integration.functions.dmlscript;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/external/DynProjectTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/external/DynProjectTest.java b/src/test/java/org/apache/sysml/test/integration/functions/external/DynProjectTest.java
index 0e965ad..0973b65 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/external/DynProjectTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/external/DynProjectTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.external;
+package org.apache.sysml.test.integration.functions.external;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class DynProjectTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/external/DynReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/external/DynReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/external/DynReadWriteTest.java
index 4870f1f..b82831a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/external/DynReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/external/DynReadWriteTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.external;
+package org.apache.sysml.test.integration.functions.external;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class DynReadWriteTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/external/FunctionExpressionsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/external/FunctionExpressionsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/external/FunctionExpressionsTest.java
index cdccbf5..b726c32 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/external/FunctionExpressionsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/external/FunctionExpressionsTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.external;
+package org.apache.sysml.test.integration.functions.external;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.utils.Statistics;
public class FunctionExpressionsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/external/OrderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/external/OrderTest.java b/src/test/java/org/apache/sysml/test/integration/functions/external/OrderTest.java
index b4cb65a..776e97c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/external/OrderTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/external/OrderTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.external;
+package org.apache.sysml.test.integration.functions.external;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class OrderTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java
index d98abab..4d1e409 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java
index 8445d18..219591c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java
index 9d2a55b..91e7ca6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java
index d2bdd0b..31a5f59 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java
index c85bad9..a7829af 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.gdfo;
+package org.apache.sysml.test.integration.functions.gdfo;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties.Format;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties.Location;
-import com.ibm.bi.dml.hops.globalopt.InterestingProperties.Partitioning;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.globalopt.InterestingProperties;
+import org.apache.sysml.hops.globalopt.InterestingProperties.Format;
+import org.apache.sysml.hops.globalopt.InterestingProperties.Location;
+import org.apache.sysml.hops.globalopt.InterestingProperties.Partitioning;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public class HashInterestingPropertiesTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/Jdk7IssueRightIndexingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/Jdk7IssueRightIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/Jdk7IssueRightIndexingTest.java
index dda988b..564bd56 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/Jdk7IssueRightIndexingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/Jdk7IssueRightIndexingTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
index 551d047..fc28119 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class LeftIndexingScalarTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
index fb42c9c..81b6938 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp.LeftIndexingMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp.LeftIndexingMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class LeftIndexingSparseDenseTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
index bee41b5..5a5497a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp.LeftIndexingMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp.LeftIndexingMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class LeftIndexingSparseSparseTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingTest.java
index b03bba7..4c92565 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp.LeftIndexingMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp.LeftIndexingMethod;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class LeftIndexingTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
index ceccf42..9a62ab4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingVectorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingVectorTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingVectorTest.java
index 9114af1..deaeb7b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingVectorTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingVectorTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.indexing;
+package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/FullDynWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/FullDynWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/FullDynWriteTest.java
index be6c54b..e50c74b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/FullDynWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/FullDynWriteTest.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullDynWriteTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest1.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest1.java b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest1.java
index ea84053..8a94029 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest1.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest1.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest2.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest2.java
index 12c59d8..1dcd5bc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest2.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest2.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest3.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest3.java b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest3.java
index c986705..f441e64 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest3.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest3.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest4.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest4.java b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest4.java
index 49594e6..3446da1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest4.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest4.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest5.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest5.java b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest5.java
index 3ccc053..109216b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest5.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/IOTest5.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.io;
+package org.apache.sysml.test.integration.functions.io;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
[21/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java b/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
deleted file mode 100644
index ed6f955..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
+++ /dev/null
@@ -1,2814 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.OpOp3;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.ipa.InterProceduralAnalysis;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.FormatType;
-import com.ibm.bi.dml.parser.Expression.ParameterizedBuiltinFunctionOp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-
-public class DMLTranslator
-{
-
- public static int DMLBlockSize = 1000;
-
- private static final Log LOG = LogFactory.getLog(DMLTranslator.class.getName());
- private DMLProgram _dmlProg = null;
-
-
- public DMLTranslator(DMLProgram dmlp)
- throws DMLRuntimeException
- {
- _dmlProg = dmlp;
-
- //each script sets its own block size, opt level etc
- DMLConfig conf = ConfigurationManager.getConfig();
- DMLTranslator.setDMLBlockSize( conf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ) );
- OptimizerUtils.setOptimizationLevel( conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL) );
- Recompiler.reinitRecompiler(); //reinit rewriter according to opt level flags
- }
-
- /**
- *
- * @param blocksize
- */
- public static void setDMLBlockSize(int blocksize) {
- DMLBlockSize = blocksize;
- }
-
- /**
- * Validate parse tree
- *
- * @throws LanguageException
- * @throws IOException
- */
- public void validateParseTree(DMLProgram dmlp)
- throws LanguageException, ParseException, IOException
- {
- //STEP1: Pre-processing steps for validate - e.g., prepare read-after-write meta data
- boolean fWriteRead = prepareReadAfterWrite(dmlp, new HashMap<String, DataIdentifier>());
-
- //STEP2: Actual Validate
- // handle functions in namespaces (current program has default namespace)
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
-
- // for each function defined in the namespace
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock fblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
-
- HashMap<String, ConstIdentifier> constVars = new HashMap<String, ConstIdentifier>();
- VariableSet vs = new VariableSet();
-
- // add the input variables for the function to input variable list
- FunctionStatement fstmt = (FunctionStatement)fblock.getStatement(0);
- if (fblock.getNumStatements() > 1){
- LOG.error(fstmt.printErrorLocation() + "FunctionStatementBlock can only have 1 FunctionStatement");
- throw new LanguageException(fstmt.printErrorLocation() + "FunctionStatementBlock can only have 1 FunctionStatement");
- }
-
- for (DataIdentifier currVar : fstmt.getInputParams()) {
-
- if (currVar.getDataType() == DataType.SCALAR){
- currVar.setDimensions(0, 0);
- }
-
- vs.addVariable(currVar.getName(), currVar);
- }
- fblock.validate(dmlp, vs, constVars, false);
- }
-
- }
-
- // handle regular blocks -- "main" program
- VariableSet vs = new VariableSet();
- HashMap<String, ConstIdentifier> constVars = new HashMap<String, ConstIdentifier>();
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- vs = sb.validate(dmlp, vs, constVars, fWriteRead);
- constVars = sb.getConstOut();
- }
-
- //STEP3: Post-processing steps after validate - e.g., prepare read-after-write meta data
- if( fWriteRead )
- {
- //propagate size and datatypes into read
- prepareReadAfterWrite(dmlp, new HashMap<String, DataIdentifier>());
-
- //re-validate main program for datatype propagation
- vs = new VariableSet();
- constVars = new HashMap<String, ConstIdentifier>();
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- vs = sb.validate(dmlp, vs, constVars, fWriteRead);
- constVars = sb.getConstOut();
- }
- }
-
- return;
- }
-
- public void liveVariableAnalysis(DMLProgram dmlp) throws LanguageException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()) {
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock fsb = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
-
- // perform function inlining
- fstmt.setBody(StatementBlock.mergeFunctionCalls(fstmt.getBody(), dmlp));
-
- VariableSet activeIn = new VariableSet();
- for (DataIdentifier id : fstmt.getInputParams()){
- activeIn.addVariable(id.getName(), id);
- }
- fsb.initializeforwardLV(activeIn);
- }
- }
-
- // for each namespace, handle function program blocks -- backward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()) {
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
-
- // add output variables to liveout / activeout set
- FunctionStatementBlock fsb = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- VariableSet currentLiveOut = new VariableSet();
- VariableSet currentLiveIn = new VariableSet();
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
-
- for (DataIdentifier id : fstmt.getInputParams())
- currentLiveIn.addVariable(id.getName(), id);
-
- for (DataIdentifier id : fstmt.getOutputParams())
- currentLiveOut.addVariable(id.getName(), id);
-
- fsb._liveOut = currentLiveOut;
- fsb.analyze(currentLiveIn, currentLiveOut);
- }
- }
-
-
- // handle regular program blocks
- VariableSet currentLiveOut = new VariableSet();
- VariableSet activeIn = new VariableSet();
-
- // handle function inlining
- dmlp.setStatementBlocks(StatementBlock.mergeFunctionCalls(dmlp.getStatementBlocks(), dmlp));
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- activeIn = sb.initializeforwardLV(activeIn);
- }
-
- if (dmlp.getNumStatementBlocks() > 0){
- StatementBlock lastSb = dmlp.getStatementBlock(dmlp.getNumStatementBlocks() - 1);
- lastSb._liveOut = new VariableSet();
- for (int i = dmlp.getNumStatementBlocks() - 1; i >= 0; i--) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- currentLiveOut = sb.analyze(currentLiveOut);
- }
- }
- return;
-
- }
-
- /**
- * Construct Hops from parse tree
- *
- * @throws ParseException
- */
- public void constructHops(DMLProgram dmlp)
- throws ParseException, LanguageException
- {
- // Step 1: construct hops for all functions
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock current = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- constructHops(current);
- }
- }
-
- // Step 2: construct hops for main program
- // handle regular program blocks
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- constructHops(current);
- }
- }
-
- /**
- *
- * @param dmlp
- * @throws ParseException
- * @throws LanguageException
- * @throws HopsException
- */
- public void rewriteHopsDAG(DMLProgram dmlp)
- throws ParseException, LanguageException, HopsException
- {
- //apply hop rewrites (static rewrites)
- ProgramRewriter rewriter = new ProgramRewriter(true, false);
- rewriter.rewriteProgramHopDAGs(dmlp);
- resetHopsDAGVisitStatus(dmlp);
-
- //propagate size information from main into functions (but conservatively)
- if( OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS ) {
- InterProceduralAnalysis ipa = new InterProceduralAnalysis();
- ipa.analyzeProgram(dmlp);
- resetHopsDAGVisitStatus(dmlp);
- }
-
- //apply hop rewrites (dynamic rewrites, after IPA)
- ProgramRewriter rewriter2 = new ProgramRewriter(false, true);
- rewriter2.rewriteProgramHopDAGs(dmlp);
- resetHopsDAGVisitStatus(dmlp);
-
- // Compute memory estimates for all the hops. These estimates are used
- // subsequently in various optimizations, e.g. CP vs. MR scheduling and parfor.
- refreshMemEstimates(dmlp);
- resetHopsDAGVisitStatus(dmlp);
- }
-
-
- public void constructLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException {
-
- // for each namespace, handle function program blocks handle function
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock current = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- constructLops(current);
- }
- }
-
- // handle regular program blocks
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- constructLops(current);
- }
- }
-
- /**
- *
- * @param sb
- * @throws HopsException
- * @throws LopsException
- */
- public void constructLops(StatementBlock sb)
- throws HopsException, LopsException
- {
- if (sb instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock)sb;
- WhileStatement whileStmt = (WhileStatement)wsb.getStatement(0);
- ArrayList<StatementBlock> body = whileStmt.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()) {
- LOG.error(sb.printBlockErrorLocation() + "WhileStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "WhileStatementBlock should not have hops");
- }
- // step through stmt blocks in while stmt body
- for (StatementBlock stmtBlock : body){
- constructLops(stmtBlock);
- }
-
- // handle while stmt predicate
- Lop l = wsb.getPredicateHops().constructLops();
- wsb.set_predicateLops(l);
- wsb.updatePredicateRecompilationFlag();
- }
-
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement ifStmt = (IfStatement)isb.getStatement(0);
- ArrayList<StatementBlock> ifBody = ifStmt.getIfBody();
- ArrayList<StatementBlock> elseBody = ifStmt.getElseBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()){
- LOG.error(sb.printBlockErrorLocation() + "IfStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "IfStatementBlock should not have hops");
- }
- // step through stmt blocks in if stmt ifBody
- for (StatementBlock stmtBlock : ifBody)
- constructLops(stmtBlock);
-
- // step through stmt blocks in if stmt elseBody
- for (StatementBlock stmtBlock : elseBody)
- constructLops(stmtBlock);
-
- // handle if stmt predicate
- Lop l = isb.getPredicateHops().constructLops();
- isb.set_predicateLops(l);
- isb.updatePredicateRecompilationFlag();
- }
-
- else if (sb instanceof ForStatementBlock) //NOTE: applies to ForStatementBlock and ParForStatementBlock
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fs = (ForStatement)sb.getStatement(0);
- ArrayList<StatementBlock> body = fs.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty() ) {
- LOG.error(sb.printBlockErrorLocation() + "ForStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "ForStatementBlock should not have hops");
- }
- // step through stmt blocks in FOR stmt body
- for (StatementBlock stmtBlock : body)
- constructLops(stmtBlock);
-
- // handle for stmt predicate
- if (fsb.getFromHops() != null){
- Lop llobs = fsb.getFromHops().constructLops();
- fsb.setFromLops(llobs);
- }
- if (fsb.getToHops() != null){
- Lop llobs = fsb.getToHops().constructLops();
- fsb.setToLops(llobs);
- }
- if (fsb.getIncrementHops() != null){
- Lop llobs = fsb.getIncrementHops().constructLops();
- fsb.setIncrementLops(llobs);
- }
- fsb.updatePredicateRecompilationFlags();
- }
- else if (sb instanceof FunctionStatementBlock){
- FunctionStatement functStmt = (FunctionStatement)sb.getStatement(0);
- ArrayList<StatementBlock> body = functStmt.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()) {
- LOG.error(sb.printBlockErrorLocation() + "FunctionStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "FunctionStatementBlock should not have hops");
- }
- // step through stmt blocks in while stmt body
- for (StatementBlock stmtBlock : body){
- constructLops(stmtBlock);
- }
- }
-
- // handle default case for regular StatementBlock
- else {
-
- if (sb.get_hops() == null)
- sb.set_hops(new ArrayList<Hop>());
-
- ArrayList<Lop> lops = new ArrayList<Lop>();
- for (Hop hop : sb.get_hops()) {
- lops.add(hop.constructLops());
- }
- sb.setLops(lops);
- sb.updateRecompilationFlag();
- }
-
- } // end method
-
-
- public void printLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException {
- if (LOG.isDebugEnabled()){
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- printLops(fsblock);
- }
- }
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- printLops(current);
- }
- }
- }
-
- public void printLops(StatementBlock current) throws ParseException, HopsException, LopsException {
- if (LOG.isDebugEnabled()){
- ArrayList<Lop> lopsDAG = current.getLops();
-
- LOG.debug("\n********************** LOPS DAG FOR BLOCK *******************");
-
- if (current instanceof FunctionStatementBlock) {
- if (current.getNumStatements() > 1)
- LOG.debug("Function statement block has more than 1 stmt");
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock child : fstmt.getBody()){
- printLops(child);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
-
- // print predicate lops
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- Hop predicateHops = ((WhileStatementBlock) current).getPredicateHops();
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- Lop predicateLops = predicateHops.getLops();
- if (predicateLops == null)
- predicateLops = predicateHops.constructLops();
- predicateLops.printMe();
-
- if (wstb.getNumStatements() > 1){
- LOG.error(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
- throw new HopsException(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
- }
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printLops(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
-
- // print predicate lops
- IfStatementBlock istb = (IfStatementBlock) current;
- Hop predicateHops = ((IfStatementBlock) current).getPredicateHops();
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- Lop predicateLops = predicateHops.getLops();
- if (predicateLops == null)
- predicateLops = predicateHops.constructLops();
- predicateLops.printMe();
-
- if (istb.getNumStatements() > 1){
- LOG.error(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
- throw new HopsException(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
- }
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
- for (StatementBlock sb : is.getIfBody()){
- printLops(sb);
- }
- if ( !is.getElseBody().isEmpty() ){
- LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
- for (StatementBlock sb : is.getElseBody()){
- printLops(sb);
- }
- }
- }
-
- if (current instanceof ForStatementBlock) {
-
- // print predicate lops
- ForStatementBlock fsb = (ForStatementBlock) current;
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- if( fsb.getFromHops() != null ){
- LOG.debug("FROM:");
- Lop llops = fsb.getFromLops();
- if( llops == null )
- llops = fsb.getFromHops().constructLops();
- llops.printMe();
- }
- if( fsb.getToHops() != null ){
- LOG.debug("TO:");
- Lop llops = fsb.getToLops();
- if( llops == null )
- llops = fsb.getToHops().constructLops();
- llops.printMe();
- }
- if( fsb.getIncrementHops() != null ){
- LOG.debug("INCREMENT:");
- Lop llops = fsb.getIncrementLops();
- if( llops == null )
- llops = fsb.getIncrementHops().constructLops();
- llops.printMe();
- }
-
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
- throw new HopsException(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
- }
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printLops(sb);
- }
- }
-
- if (lopsDAG != null && !lopsDAG.isEmpty() ) {
- Iterator<Lop> iter = lopsDAG.iterator();
- while (iter.hasNext()) {
- LOG.debug("\n********************** OUTPUT LOPS *******************");
- iter.next().printMe();
- }
- }
- }
- }
-
-
- public void printHops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
- if (LOG.isDebugEnabled()) {
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- printHops(fsblock);
- }
- }
-
- // hand
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- printHops(current);
- }
- }
- }
-
- public void printHops(StatementBlock current) throws ParseException, HopsException {
- if (LOG.isDebugEnabled()) {
- ArrayList<Hop> hopsDAG = current.get_hops();
- LOG.debug("\n********************** HOPS DAG FOR BLOCK *******************");
-
- if (current instanceof FunctionStatementBlock) {
- if (current.getNumStatements() > 1)
- LOG.debug("Function statement block has more than 1 stmt");
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock child : fstmt.getBody()){
- printHops(child);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
-
- // print predicate hops
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- Hop predicateHops = wstb.getPredicateHops();
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- predicateHops.printMe();
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printHops(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
-
- // print predicate hops
- IfStatementBlock istb = (IfStatementBlock) current;
- Hop predicateHops = istb.getPredicateHops();
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- predicateHops.printMe();
-
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- printHops(sb);
- }
-
- for (StatementBlock sb : is.getElseBody()){
- printHops(sb);
- }
- }
-
-
- if (current instanceof ForStatementBlock) {
-
- // print predicate hops
- ForStatementBlock fsb = (ForStatementBlock) current;
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- if (fsb.getFromHops() != null) fsb.getFromHops().printMe();
- if (fsb.getToHops() != null) fsb.getToHops().printMe();
- if (fsb.getIncrementHops() != null) fsb.getIncrementHops().printMe();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printHops(sb);
- }
- }
-
- if (hopsDAG != null && !hopsDAG.isEmpty()) {
- // hopsDAG.iterator().next().printMe();
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()) {
- LOG.debug("\n********************** OUTPUT HOPS *******************");
- iter.next().printMe();
- }
- }
- }
- }
-
- public void refreshMemEstimates(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- refreshMemEstimates(fsblock);
- }
- }
-
- // handle statement blocks in "main" method
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- refreshMemEstimates(current);
- }
- }
-
- public void refreshMemEstimates(StatementBlock current) throws ParseException, HopsException {
-
- MemoTable memo = new MemoTable();
- ArrayList<Hop> hopsDAG = current.get_hops();
- if (hopsDAG != null && !hopsDAG.isEmpty()) {
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()) {
- iter.next().refreshMemEstimates(memo);
- }
- }
-
- if (current instanceof FunctionStatementBlock) {
-
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock sb : fstmt.getBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
- // handle predicate
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.getPredicateHops().refreshMemEstimates(new MemoTable());
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- // handle predicate
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.getPredicateHops().refreshMemEstimates(new MemoTable());
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- refreshMemEstimates(sb);
- }
- for (StatementBlock sb : is.getElseBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- // handle predicate
- ForStatementBlock fsb = (ForStatementBlock) current;
- if (fsb.getFromHops() != null)
- fsb.getFromHops().refreshMemEstimates(new MemoTable());
- if (fsb.getToHops() != null)
- fsb.getToHops().refreshMemEstimates(new MemoTable());
- if (fsb.getIncrementHops() != null)
- fsb.getIncrementHops().refreshMemEstimates(new MemoTable());
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- refreshMemEstimates(sb);
- }
- }
- }
-
- public static void resetHopsDAGVisitStatus(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- resetHopsDAGVisitStatus(fsblock);
- }
- }
-
- // handle statement blocks in "main" method
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- resetHopsDAGVisitStatus(current);
- }
- }
-
- public static void resetHopsDAGVisitStatus(StatementBlock current) throws ParseException, HopsException {
-
- ArrayList<Hop> hopsDAG = current.get_hops();
- if (hopsDAG != null && !hopsDAG.isEmpty() ) {
- Hop.resetVisitStatus(hopsDAG);
- }
-
- if (current instanceof FunctionStatementBlock) {
-
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock sb : fstmt.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
- // handle predicate
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.getPredicateHops().resetVisitStatus();
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While stmt block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- // handle predicate
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.getPredicateHops().resetVisitStatus();
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- resetHopsDAGVisitStatus(sb);
- }
- for (StatementBlock sb : is.getElseBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- // handle predicate
- ForStatementBlock fsb = (ForStatementBlock) current;
- if (fsb.getFromHops() != null)
- fsb.getFromHops().resetVisitStatus();
- if (fsb.getToHops() != null)
- fsb.getToHops().resetVisitStatus();
- if (fsb.getIncrementHops() != null)
- fsb.getIncrementHops().resetVisitStatus();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statment block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
- }
-
- public void resetLopsDAGVisitStatus(DMLProgram dmlp) throws HopsException, LanguageException {
-
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- resetLopsDAGVisitStatus(fsblock);
- }
- }
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- resetLopsDAGVisitStatus(current);
- }
- }
-
- public void resetLopsDAGVisitStatus(StatementBlock current) throws HopsException {
-
- ArrayList<Hop> hopsDAG = current.get_hops();
-
- if (hopsDAG != null && !hopsDAG.isEmpty() ) {
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()){
- Hop currentHop = iter.next();
- currentHop.getLops().resetVisitStatus();
- }
- }
-
- if (current instanceof FunctionStatementBlock) {
- FunctionStatementBlock fsb = (FunctionStatementBlock) current;
- FunctionStatement fs = (FunctionStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : fs.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
-
- if (current instanceof WhileStatementBlock) {
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.get_predicateLops().resetVisitStatus();
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.get_predicateLops().resetVisitStatus();
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- resetLopsDAGVisitStatus(sb);
- }
-
- for (StatementBlock sb : is.getElseBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- ForStatementBlock fsb = (ForStatementBlock) current;
-
- if (fsb.getFromLops() != null)
- fsb.getFromLops().resetVisitStatus();
- if (fsb.getToLops() != null)
- fsb.getToLops().resetVisitStatus();
- if (fsb.getIncrementLops() != null)
- fsb.getIncrementLops().resetVisitStatus();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
- }
-
-
- public void constructHops(StatementBlock sb)
- throws ParseException, LanguageException {
-
- if (sb instanceof WhileStatementBlock) {
- constructHopsForWhileControlBlock((WhileStatementBlock) sb);
- return;
- }
-
- if (sb instanceof IfStatementBlock) {
- constructHopsForIfControlBlock((IfStatementBlock) sb);
- return;
- }
-
- if (sb instanceof ForStatementBlock) { //NOTE: applies to ForStatementBlock and ParForStatementBlock
- constructHopsForForControlBlock((ForStatementBlock) sb);
- return;
- }
-
- if (sb instanceof FunctionStatementBlock) {
- constructHopsForFunctionControlBlock((FunctionStatementBlock) sb);
- return;
- }
-
-
- HashMap<String, Hop> ids = new HashMap<String, Hop>();
- ArrayList<Hop> output = new ArrayList<Hop>();
-
- VariableSet liveIn = sb.liveIn();
- VariableSet liveOut = sb.liveOut();
- VariableSet updated = sb._updated;
- VariableSet gen = sb._gen;
- VariableSet updatedLiveOut = new VariableSet();
-
- // handle liveout variables that are updated --> target identifiers for Assignment
- HashMap<String, Integer> liveOutToTemp = new HashMap<String, Integer>();
- for (int i = 0; i < sb.getNumStatements(); i++) {
- Statement current = sb.getStatement(i);
-
- if (current instanceof AssignmentStatement) {
- AssignmentStatement as = (AssignmentStatement) current;
- DataIdentifier target = as.getTarget();
- if (liveOut.containsVariable(target.getName())) {
- liveOutToTemp.put(target.getName(), Integer.valueOf(i));
- }
- }
- if (current instanceof MultiAssignmentStatement) {
- MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
-
- for (DataIdentifier target : mas.getTargetList()){
- if (liveOut.containsVariable(target.getName())) {
- liveOutToTemp.put(target.getName(), Integer.valueOf(i));
- }
- }
- }
- }
-
- // only create transient read operations for variables either updated or read-before-update
- // (i.e., from LV analysis, updated and gen sets)
- if ( !liveIn.getVariables().values().isEmpty() ) {
-
- for (String varName : liveIn.getVariables().keySet()) {
-
- if (updated.containsVariable(varName) || gen.containsVariable(varName)){
-
- DataIdentifier var = liveIn.getVariables().get(varName);
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- ids.put(varName, read);
- }
- }
- }
-
-
- for( int i = 0; i < sb.getNumStatements(); i++ ) {
- Statement current = sb.getStatement(i);
-
- if (current instanceof OutputStatement) {
- OutputStatement os = (OutputStatement) current;
-
- DataExpression source = os.getSource();
- DataIdentifier target = os.getIdentifier();
-
- //error handling unsupported indexing expression in write statement
- if( target instanceof IndexedIdentifier ) {
- throw new LanguageException(source.printErrorLocation()+": Unsupported indexing expression in write statement. " +
- "Please, assign the right indexing result to a variable and write this variable.");
- }
-
- DataOp ae = (DataOp)processExpression(source, target, ids);
- String formatName = os.getExprParam(DataExpression.FORMAT_TYPE).toString();
- ae.setInputFormatType(Expression.convertFormatType(formatName));
-
- if (ae.getDataType() == DataType.SCALAR ) {
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), -1, -1);
- }
- else {
- switch(ae.getInputFormatType()) {
- case TEXT:
- case MM:
- case CSV:
- // write output in textcell format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), -1, -1);
- break;
-
- case BINARY:
- // write output in binary block format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- break;
-
- default:
- throw new LanguageException("Unrecognized file format: " + ae.getInputFormatType());
- }
- }
-
- output.add(ae);
-
- }
-
- if (current instanceof PrintStatement) {
- PrintStatement ps = (PrintStatement) current;
- Expression source = ps.getExpression();
- PRINTTYPE ptype = ps.getType();
-
- DataIdentifier target = createTarget();
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.STRING);
- target.setAllPositions(current.getFilename(), current.getBeginLine(), target.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- Hop ae = processExpression(source, target, ids);
-
- try {
- Hop.OpOp1 op = (ptype == PRINTTYPE.PRINT ? Hop.OpOp1.PRINT : Hop.OpOp1.STOP);
- Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
- printHop.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
- output.add(printHop);
- } catch ( HopsException e ) {
- throw new LanguageException(e);
- }
- }
-
- if (current instanceof AssignmentStatement) {
-
- AssignmentStatement as = (AssignmentStatement) current;
- DataIdentifier target = as.getTarget();
- Expression source = as.getSource();
-
-
- // CASE: regular assignment statement -- source is DML expression that is NOT user-defined or external function
- if (!(source instanceof FunctionCallIdentifier)){
-
- // CASE: target is regular data identifier
- if (!(target instanceof IndexedIdentifier)) {
-
- Hop ae = processExpression(source, target, ids);
- ids.put(target.getName(), ae);
- target.setProperties(source.getOutput());
- Integer statementId = liveOutToTemp.get(target.getName());
- if ((statementId != null) && (statementId.intValue() == i)) {
- DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
- transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getRowsInBlock(), ae.getColsInBlock());
- transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndLine());
- updatedLiveOut.addVariable(target.getName(), target);
- output.add(transientwrite);
- }
- } // end if (!(target instanceof IndexedIdentifier)) {
-
- // CASE: target is indexed identifier (left-hand side indexed expression)
- else {
- Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier)target, ids);
-
- ids.put(target.getName(), ae);
-
- // obtain origDim values BEFORE they are potentially updated during setProperties call
- // (this is incorrect for LHS Indexing)
- long origDim1 = ((IndexedIdentifier)target).getOrigDim1();
- long origDim2 = ((IndexedIdentifier)target).getOrigDim2();
- target.setProperties(source.getOutput());
- ((IndexedIdentifier)target).setOriginalDimensions(origDim1, origDim2);
-
- // preserve data type matrix of any index identifier
- // (required for scalar input to left indexing)
- // TODO Doug, please verify this (required for scalar assignments, otherwise TWrite of LIX target becomes scalar as well)
- if( target.getDataType() != DataType.MATRIX ) {
- target.setDataType(DataType.MATRIX);
- target.setValueType(ValueType.DOUBLE);
- target.setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- }
-
- Integer statementId = liveOutToTemp.get(target.getName());
- if ((statementId != null) && (statementId.intValue() == i)) {
- DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
- transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getRowsInBlock(), ae.getColsInBlock());
- transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- updatedLiveOut.addVariable(target.getName(), target);
- output.add(transientwrite);
- }
- }
-
-
- }
- else
- {
- //assignment, function call
- FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
- FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName());
-
- //error handling missing function
- if (fsb == null){
- String error = source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace();
- LOG.error(error);
- throw new LanguageException(error);
- }
-
- //error handling unsupported function call in indexing expression
- if( target instanceof IndexedIdentifier ) {
- String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(),fci.getName());
- throw new LanguageException("Unsupported function call to '"+fkey+"' in left indexing expression. " +
- "Please, assign the function output to a variable.");
- }
-
- ArrayList<Hop> finputs = new ArrayList<Hop>();
- for (ParameterExpression paramName : fci.getParamExprs()){
- Hop in = processExpression(paramName.getExpr(), null, ids);
- finputs.add(in);
- }
-
- //create function op
- FunctionType ftype = fsb.getFunctionOpType();
- FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{target.getName()});
- output.add(fcall);
-
- //TODO function output dataops (phase 3)
- //DataOp trFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), fcall, DataOpTypes.FUNCTIONOUTPUT, null);
- //DataOp twFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), trFoutput, DataOpTypes.TRANSIENTWRITE, null);
- }
- }
-
- else if (current instanceof MultiAssignmentStatement) {
- //multi-assignment, by definition a function call
- MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
- Expression source = mas.getSource();
-
- if ( source instanceof FunctionCallIdentifier ) {
- FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
- FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName());
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- if (fstmt == null){
- LOG.error(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
- throw new LanguageException(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
- }
-
- ArrayList<Hop> finputs = new ArrayList<Hop>();
- for (ParameterExpression paramName : fci.getParamExprs()){
- Hop in = processExpression(paramName.getExpr(), null, ids);
- finputs.add(in);
- }
-
- //create function op
- String[] foutputs = new String[mas.getTargetList().size()];
- int count = 0;
- for ( DataIdentifier paramName : mas.getTargetList() ){
- foutputs[count++]=paramName.getName();
- }
-
- FunctionType ftype = fsb.getFunctionOpType();
- FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs);
- output.add(fcall);
-
- //TODO function output dataops (phase 3)
- /*for ( DataIdentifier paramName : mas.getTargetList() ){
- DataOp twFoutput = new DataOp(paramName.getName(), paramName.getDataType(), paramName.getValueType(), fcall, DataOpTypes.TRANSIENTWRITE, null);
- output.add(twFoutput);
- }*/
- }
- else if ( source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression)source).multipleReturns() ) {
- // construct input hops
- Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression)source, mas.getTargetList(), ids);
- output.add(fcall);
-
- }
- else
- throw new LanguageException("Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements");
- }
-
- }
- sb.updateLiveVariablesOut(updatedLiveOut);
- sb.set_hops(output);
-
- }
-
- public void constructHopsForIfControlBlock(IfStatementBlock sb) throws ParseException, LanguageException {
-
- IfStatement ifsb = (IfStatement) sb.getStatement(0);
- ArrayList<StatementBlock> ifBody = ifsb.getIfBody();
- ArrayList<StatementBlock> elseBody = ifsb.getElseBody();
-
- // construct hops for predicate in if statement
- constructHopsForConditionalPredicate(sb);
-
- // handle if statement body
- for( StatementBlock current : ifBody ) {
- constructHops(current);
- }
-
- // handle else stmt body
- for( StatementBlock current : elseBody ) {
- constructHops(current);
- }
- }
-
- /**
- * Constructs Hops for a given ForStatementBlock or ParForStatementBlock, respectively.
- *
- * @param sb
- * @throws ParseException
- * @throws LanguageException
- */
- public void constructHopsForForControlBlock(ForStatementBlock sb)
- throws ParseException, LanguageException
- {
-
- ForStatement fs = (ForStatement) sb.getStatement(0);
- ArrayList<StatementBlock> body = fs.getBody();
-
- // construct hops for iterable predicate
- constructHopsForIterablePredicate(sb);
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
- public void constructHopsForFunctionControlBlock(FunctionStatementBlock fsb) throws ParseException, LanguageException {
-
- ArrayList<StatementBlock> body = ((FunctionStatement)fsb.getStatement(0)).getBody();
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
- public void constructHopsForWhileControlBlock(WhileStatementBlock sb)
- throws ParseException, LanguageException {
-
- ArrayList<StatementBlock> body = ((WhileStatement)sb.getStatement(0)).getBody();
-
- // construct hops for while predicate
- constructHopsForConditionalPredicate(sb);
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
-
- public void constructHopsForConditionalPredicate(StatementBlock passedSB) throws ParseException {
-
- HashMap<String, Hop> _ids = new HashMap<String, Hop>();
-
- // set conditional predicate
- ConditionalPredicate cp = null;
-
- if (passedSB instanceof WhileStatementBlock){
- WhileStatement ws = (WhileStatement) ((WhileStatementBlock)passedSB).getStatement(0);
- cp = ws.getConditionalPredicate();
- }
- else if (passedSB instanceof IfStatementBlock) {
- IfStatement ws = (IfStatement) ((IfStatementBlock)passedSB).getStatement(0);
- cp = ws.getConditionalPredicate();
- }
- else {
- throw new ParseException("ConditionalPredicate expected only for while or if statements.");
- }
-
- VariableSet varsRead = cp.variablesRead();
-
- for (String varName : varsRead.getVariables().keySet()) {
-
- // creating transient read for live in variables
- DataIdentifier var = passedSB.liveIn().getVariables().get(varName);
-
- DataOp read = null;
-
- if (var == null) {
- LOG.error("variable " + varName + " not live variable for conditional predicate");
- throw new ParseException("variable " + varName + " not live variable for conditional predicate");
- } else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
-
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
-
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.BOOLEAN);
- target.setAllPositions(passedSB.getFilename(), passedSB.getBeginLine(), passedSB.getBeginColumn(), passedSB.getEndLine(), passedSB.getEndColumn());
- Hop predicateHops = null;
- Expression predicate = cp.getPredicate();
-
- if (predicate instanceof RelationalExpression) {
- predicateHops = processRelationalExpression((RelationalExpression) cp.getPredicate(), target, _ids);
- } else if (predicate instanceof BooleanExpression) {
- predicateHops = processBooleanExpression((BooleanExpression) cp.getPredicate(), target, _ids);
- } else if (predicate instanceof DataIdentifier) {
- // handle data identifier predicate
- predicateHops = processExpression(cp.getPredicate(), null, _ids);
- } else if (predicate instanceof ConstIdentifier) {
- // handle constant identifier
- // a) translate 0 --> FALSE; translate 1 --> TRUE
- // b) disallow string values
- if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 0) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 0.0)) {
- cp.setPredicate(new BooleanIdentifier(false,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
-
- }
- else if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 1) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 1.0)) {
- cp.setPredicate(new BooleanIdentifier(true,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
- }
- else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier){
- cp.setPredicate(new BooleanIdentifier(true,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
- LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + "' (!= 0/1) is converted to boolean TRUE by DML");
- }
- else if (predicate instanceof StringIdentifier) {
- LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
- throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
-
- }
- predicateHops = processExpression(cp.getPredicate(), null, _ids);
- }
- if (passedSB instanceof WhileStatementBlock)
- ((WhileStatementBlock)passedSB).setPredicateHops(predicateHops);
- else if (passedSB instanceof IfStatementBlock)
- ((IfStatementBlock)passedSB).setPredicateHops(predicateHops);
- }
-
-
- /**
- * Constructs all predicate Hops (for FROM, TO, INCREMENT) of an iterable predicate
- * and assigns these Hops to the passed statement block.
- *
- * Method used for both ForStatementBlock and ParForStatementBlock.
- *
- * @param passedSB
- * @throws ParseException
- */
- public void constructHopsForIterablePredicate(ForStatementBlock fsb)
- throws ParseException
- {
- HashMap<String, Hop> _ids = new HashMap<String, Hop>();
-
- // set iterable predicate
- ForStatement fs = (ForStatement) fsb.getStatement(0);
- IterablePredicate ip = fs.getIterablePredicate();
-
- for(int i=0; i < 3; i++) {
- VariableSet varsRead = null;
- if (i==0)
- varsRead = ip.getFromExpr().variablesRead();
- else if (i==1)
- varsRead = ip.getToExpr().variablesRead();
- else
- varsRead = ip.getIncrementExpr().variablesRead();
-
- if(varsRead != null) {
- for (String varName : varsRead.getVariables().keySet()) {
-
- DataIdentifier var = fsb.liveIn().getVariable(varName);
- DataOp read = null;
- if (var == null) {
- LOG.error("variable '" + varName + "' is not available for iterable predicate");
- throw new ParseException("variable '" + varName + "' is not available for iterable predicate");
- }
- else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
- }
-
- //construct hops for from, to, and increment expressions
- if(i==0)
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- else if(i==1)
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
- else
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));
-
- }
-
- /*VariableSet varsRead = ip.variablesRead();
-
- for (String varName : varsRead.getVariables().keySet()) {
-
- DataIdentifier var = passedSB.liveIn().getVariable(varName);
- DataOp read = null;
- if (var == null) {
- LOG.error(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- throw new ParseException(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- }
- else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
-
- //construct hops for from, to, and increment expressions
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));*/
- }
-
-
- /**
- * Construct Hops from parse tree : Process Expression in an assignment
- * statement
- *
- * @throws ParseException
- */
- private Hop processExpression(Expression source, DataIdentifier target, HashMap<String, Hop> hops) throws ParseException {
- if (source.getKind() == Expression.Kind.BinaryOp) {
- return processBinaryExpression((BinaryExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.RelationalOp) {
- return processRelationalExpression((RelationalExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.BooleanOp) {
- return processBooleanExpression((BooleanExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.Data) {
- if (source instanceof IndexedIdentifier){
- IndexedIdentifier sourceIndexed = (IndexedIdentifier) source;
- return processIndexingExpression(sourceIndexed,target,hops);
- } else if (source instanceof IntIdentifier) {
- IntIdentifier sourceInt = (IntIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceInt.getValue());
- litop.setAllPositions(sourceInt.getBeginLine(), sourceInt.getBeginColumn(), sourceInt.getEndLine(), sourceInt.getEndColumn());
- setIdentifierParams(litop, sourceInt);
- return litop;
- } else if (source instanceof DoubleIdentifier) {
- DoubleIdentifier sourceDouble = (DoubleIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceDouble.getValue());
- litop.setAllPositions(sourceDouble.getBeginLine(), sourceDouble.getBeginColumn(), sourceDouble.getEndLine(), sourceDouble.getEndColumn());
- setIdentifierParams(litop, sourceDouble);
- return litop;
- } else if (source instanceof DataIdentifier) {
- DataIdentifier sourceId = (DataIdentifier) source;
- return hops.get(sourceId.getName());
- } else if (source instanceof BooleanIdentifier) {
- BooleanIdentifier sourceBoolean = (BooleanIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceBoolean.getValue());
- litop.setAllPositions(sourceBoolean.getBeginLine(), sourceBoolean.getBeginColumn(), sourceBoolean.getEndLine(), sourceBoolean.getEndColumn());
- setIdentifierParams(litop, sourceBoolean);
- return litop;
- } else if (source instanceof StringIdentifier) {
- StringIdentifier sourceString = (StringIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceString.getValue());
- litop.setAllPositions(sourceString.getBeginLine(), sourceString.getBeginColumn(), sourceString.getEndLine(), sourceString.getEndColumn());
- setIdentifierParams(litop, sourceString);
- return litop;
- }
- } else if (source.getKind() == Expression.Kind.BuiltinFunctionOp) {
- try {
- return processBuiltinFunctionExpression((BuiltinFunctionExpression) source, target, hops);
- } catch (HopsException e) {
- throw new ParseException(e.getMessage());
- }
- } else if (source.getKind() == Expression.Kind.ParameterizedBuiltinFunctionOp ) {
- try {
- return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression)source, target, hops);
- } catch ( HopsException e ) {
- throw new ParseException(e.getMessage());
- }
- } else if (source.getKind() == Expression.Kind.DataOp ) {
- try {
- Hop ae = (Hop)processDataExpression((DataExpression)source, target, hops);
-
- if (ae instanceof DataOp){
- String formatName = ((DataExpression)source).getVarParam(DataExpression.FORMAT_TYPE).toString();
- ((DataOp)ae).setInputFormatType(Expression.convertFormatType(formatName));
- }
- //hops.put(target.getName(), ae);
- return ae;
- } catch ( Exception e ) {
- throw new ParseException(e.getMessage());
- }
- }
- return null;
- } // end method processExpression
-
- private DataIdentifier createTarget(Expression source) {
- Identifier id = source.getOutput();
- if (id instanceof DataIdentifier && !(id instanceof DataExpression))
- return (DataIdentifier) id;
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- target.setProperties(id);
- return target;
- }
-
- private DataIdentifier createTarget() {
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- return target;
- }
-
-
- /**
- * Constructs the Hops for arbitrary expressions that eventually evaluate to an INT scalar.
- *
- * @param source
- * @param hops
- * @return
- * @throws ParseException
- */
- private Hop processTempIntExpression( Expression source, HashMap<String, Hop> hops )
- throws ParseException
- {
- DataIdentifier tmpOut = createTarget();
- tmpOut.setDataType(DataType.SCALAR);
- tmpOut.setValueType(ValueType.INT);
- source.setOutput(tmpOut);
-
- return processExpression(source, tmpOut, hops );
- }
-
- private Hop processLeftIndexedExpression(Expression source, IndexedIdentifier target, HashMap<String, Hop> hops)
- throws ParseException {
-
- // process target indexed expressions
- Hop rowLowerHops = null, rowUpperHops = null, colLowerHops = null, colUpperHops = null;
-
- if (target.getRowLowerBound() != null)
- rowLowerHops = processExpression(target.getRowLowerBound(),null,hops);
- else
- rowLowerHops = new LiteralOp(1);
-
- if (target.getRowUpperBound() != null)
- rowUpperHops = processExpression(target.getRowUpperBound(),null,hops);
- else
- {
- if ( target.getDim1() != -1 )
- rowUpperHops = new LiteralOp(target.getOrigDim1());
- else
- {
- try {
- //currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hops.OpOp1.NROW, expr);
- rowUpperHops = new UnaryOp(target.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(target.getName()));
- rowUpperHops.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- } catch (HopsException e) {
- LOG.error(target.printErrorLocation() + "error processing row upper index for indexed expression " + target.toString());
- throw new RuntimeException(target.printErrorLocation() + "error processing row upper index for indexed expression " + target.toString());
- }
- }
- }
- if (target.getColLowerBound() != null)
- colLowerHops = processExpression(target.getColLowerBound(),null,hops);
- else
- colLowerHops = new LiteralOp(1);
-
- if (target.getColUpperBound() != null)
- colUpperHops = processExpression(target.getColUpperBound(),null,hops);
- else
- {
- if ( target.getDim2() != -1 )
- colUpperHops = new LiteralOp(target.getOrigDim2());
- else
- {
- try {
- colUpperHops = new UnaryOp(target.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NCOL, hops.get(target.getName()));
- } catch (HopsException e) {
- LOG.error(target.printErrorLocation() + " error processing column upper index for indexed expression " + target.toString());
- throw new RuntimeException(target.printErrorLocation() + " error processing column upper index for indexed expression " + target.toString(), e);
- }
- }
- }
-
- //if (target == null) {
- // target = createTarget(source);
- //}
-
- // process the source expression to get source Hops
- Hop sourceOp = processExpression(source, target, hops);
-
- // process the target to get targetHops
- Hop targetOp = hops.get(target.getName());
- if (targetOp == null){
- LOG.error(target.printErrorLocation() + " must define matrix " + target.getName() + " before indexing operations are allowed ");
- throw new ParseException(target.printErrorLocation() + " must define matrix " + target.getName() + " before indexing operations are allowed ");
- }
-
- //TODO Doug, please verify this (we need probably a cleaner way than this postprocessing)
- if( sourceOp.getDataType() == DataType.MATRIX && source.getOutput().getDataType() == DataType.SCALAR )
- sourceOp.setDataType(DataType.SCALAR);
-
- Hop leftIndexOp = new LeftIndexingOp(target.getName(), target.getDataType(), target.getValueType(),
- targetOp, sourceOp, rowLowerHops, rowUpperHops, colLowerHops, colUpperHops,
- target.getRowLowerEqualsUpper(), target.getColLowerEqualsUpper());
-
- setIdentifierParams(leftIndexOp, target);
-
- leftIndexOp.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- leftIndexOp.setDim1(target.getOrigDim1());
- leftIndexOp.setDim2(target.getOrigDim2());
-
- return leftIndexOp;
- }
-
-
- private Hop processIndexingExpression(IndexedIdentifier source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException {
-
- // process Hops for indexes (for source)
- Hop rowLowerHops = null, rowUpperHops = null, colLowerHops = null, colUpperHops = null;
-
- if (source.getRowLowerBound() != null)
- rowLowerHops = processExpression(source.getRowLowerBound(),null,hops);
- else
- rowLowerHops = new LiteralOp(1);
-
- if (source.getRowUpperBound() != null)
- rowUpperHops = processExpression(source.getRowUpperBound(),null,hops);
- else
- {
- if ( source.getOrigDim1() != -1 )
- rowUpperHops = new LiteralOp(source.getOrigDim1());
- else
- {
- try {
- //currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hops.OpOp1.NROW, expr);
- rowUpperHops = new UnaryOp(source.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(source.getName()));
- rowUpperHops.setAllPositions(source.getBeginLine(),source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- } catch (HopsException e) {
- LOG.error(source.printErrorLocation() + "error processing row upper index for indexed identifier " + source.toString());
- throw new RuntimeException(source.printErrorLocation() + "error processing row upper index for indexed identifier " + source.toString() + e);
- }
- }
- }
- if (source.getColLowerBound() != null)
- colLowerHops = processExpression(source.getColLowerBound(),null,hops);
- else
- colLowerHops = new LiteralOp(1);
-
- if (source.getColUpperBound() != null)
- colUpperHops = processExpression(source.getColUpperBound(),null,hops);
- else
- {
- if ( source.getOrigDim2() != -1 )
- colUpperHops = new LiteralOp(source.getOrigDim2());
- else
- {
- try {
- colUpperHops = new UnaryOp(source.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NCOL, hops.get(source.getName()));
- } catch (HopsException e) {
- LOG.error(source.printErrorLocation() + "error processing column upper index for indexed indentifier " + source.toString(), e);
- throw new RuntimeException(source.printErrorLocation() + "error processing column upper index for indexed indentifier " + source.toString(), e);
- }
- }
- }
-
- if (target == null) {
- target = createTarget(source);
- }
- //unknown nnz after range indexing (applies to indexing op but also
- //data dependent operations)
- target.setNnz(-1);
-
- Hop indexOp = new IndexingOp(target.getName(), target.getDataType(), target.getValueType(),
- hops.get(source.getName()), rowLowerHops, rowUpperHops, colLowerHops, colUpperHops,
- source.getRowLowerEqualsUpper(), source.getColLowerEqualsUpper());
-
- indexOp.setAllPositions(indexOp.getBeginLine(), indexOp.getBeginColumn(), indexOp.getEndLine(), indexOp.getEndColumn());
- setIdentifierParams(indexOp, target);
-
- return indexOp;
- }
-
-
- /**
- * Construct Hops from parse tree : Process Binary Expression in an
- * assignment statement
- *
- * @throws ParseException
- */
- private Hop processBinaryExpression(BinaryExpression source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException
- {
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = processExpression(source.getRight(), null, hops);
-
- if (left == null || right == null){
- left = processExpression(source.getLeft(), null, hops);
- right = processExpression(source.getRight(), null, hops);
- }
-
- Hop currBop = null;
-
- //prepare target identifier and ensure that output type is of inferred type
- //(type should not be determined by target (e.g., string for print)
- if (target == null) {
- target = createTarget(source);
- }
- target.setValueType(source.getOutput().getValueType());
-
- if (source.getOpCode() == Expression.BinaryOp.PLUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.PLUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MINUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MINUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MULT) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.DIV) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.DIV, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MODULUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MODULUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.INTDIV) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.INTDIV, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MATMULT) {
- currBop = new AggBinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, AggOp.SUM, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.POW) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.POW, left, right);
- }
- else {
- throw new ParseException("Unsupported parsing of binary expression: "+source.getOpCode());
- }
- setIdentifierParams(currBop, source.getOutput());
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- return currBop;
-
- }
-
- private Hop processRelationalExpression(RelationalExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException {
-
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = processExpression(source.getRight(), null, hops);
-
- Hop currBop = null;
-
- if (target == null) {
- target = createTarget(source);
- if(left.getDataType() == DataType.MATRIX || right.getDataType() == DataType.MATRIX) {
- // Added to support matrix relational comparison
- target.setDataType(DataType.MATRIX);
- target.setValueType(ValueType.BOOLEAN);
- }
- else {
- // Added to support scalar relational comparison
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.BOOLEAN);
- }
- }
-
- OpOp2 op = null;
-
- if (source.getOpCode() == Expression.RelationalOp.LESS) {
- op = OpOp2.LESS;
- } else if (source.getOpCode() == Expression.RelationalOp.LESSEQUAL) {
- op = OpOp2.LESSEQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.GREATER) {
- op = OpOp2.GREATER;
- } else if (source.getOpCode() == Expression.RelationalOp.GREATEREQUAL) {
- op = OpOp2.GREATEREQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.EQUAL) {
- op = OpOp2.EQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.NOTEQUAL) {
- op = OpOp2.NOTEQUAL;
- }
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right);
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- return currBop;
- }
-
- /**
- *
- * @param source
- * @param target
- * @param hops
- * @return
- * @throws ParseException
- */
- private Hop processBooleanExpression(BooleanExpression source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException
- {
- // Boolean Not has a single parameter
- boolean constLeft = (source.getLeft().getOutput() instanceof ConstIdentifier);
- boolean constRight = false;
- if (source.getRight() != null) {
- constRight = (source.getRight().getOutput() instanceof ConstIdentifier);
- }
-
- if (constLeft || constRight) {
- LOG.error(source.printErrorLocation() + "Boolean expression with constant unsupported");
- throw new RuntimeException(source.printErrorLocation() + "Boolean expression with constant unsupported");
- }
-
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = null;
- if (source.getRight() != null) {
- right = processExpression(source.getRight(), null, hops);
- }
-
- //prepare target identifier and ensure that output type is boolean
- //(type should not be determined by target (e.g., string for print)
- if (target == null) {
- target = createTarget(source);
- }
- target.setValueType(ValueType.BOOLEAN);
-
- if (source.getRight() == null) {
- Hop currUop = null;
- try {
- currUop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NOT, left);
- currUop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- } catch (HopsException e) {
- throw new ParseException(e.getMessage());
- }
- return currUop;
- } else {
- Hop currBop = null;
- OpOp2 op = null;
-
- if (source.getOpCode() == Expression.BooleanOp.LOGICALAND) {
- op = OpOp2.AND;
- } else if (source.getOpCode() == Expression.BooleanOp.LOGICALOR) {
- op = OpOp2.OR;
- } else {
- LOG.error(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode());
- throw new RuntimeException(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode());
- }
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right);
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- // setIdentifierParams(currBop,source.getOutput());
- return currBop;
- }
- }
-
- private Hop constructDfHop(String name, DataType dt, ValueType vt, ParameterizedBuiltinFunctionOp op, HashMap<String,Hop> paramHops) throws HopsException {
-
- // Add a hop to paramHops to store distribution information.
- // Distribution parameter hops would have been already present in paramHops.
- Hop distLop = null;
- switch(op) {
- case QNORM:
- case PNORM:
- distLop = new LiteralOp("normal");
- break;
- case QT:
- case PT:
- distLop = new LiteralOp("t");
- break;
- case QF:
- case PF:
- distLop = new LiteralOp("f");
- break;
- case QCHISQ:
- case PCHISQ:
- distLop = new LiteralOp("chisq");
- break;
- case QEXP:
- case PEXP:
- distLop = new LiteralOp("exp");
- break;
-
- case CDF:
- case INVCDF:
- break;
-
- default:
- throw new HopsException("Invalid operation: " + op);
- }
- if (distLop != null)
- paramHops.put("dist", distLop);
-
- return new ParameterizedBuiltinOp(name, dt, vt, ParameterizedBuiltinFunctionExpression.pbHopMap.get(op), paramHops);
- }
-
- /**
- * Construct Hops from parse tree : Process ParameterizedBuiltinFunction Expression in an
- * assignment statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
-
- // this expression has multiple "named" parameters
- HashMap<String, Hop> paramHops = new HashMap<String,Hop>();
-
- // -- construct hops for all input parameters
- // -- store them in hashmap so that their "name"s are maintained
- Hop pHop = null;
- for ( String paramName : source.getVarParams().keySet() ) {
- pHop = processExpression(source.getVarParam(paramName), null, hops);
- paramHops.put(paramName, pHop);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // construct hop based on opcode
- switch(source.getOpCode()) {
- case CDF:
- case INVCDF:
- case QNORM:
- case QT:
- case QF:
- case QCHISQ:
- case QEXP:
- case PNORM:
- case PT:
- case PF:
- case PCHISQ:
- case PEXP:
- currBuiltinOp = constructDfHop(target.getName(), target.getDataType(), target.getValueType(), source.getOpCode(), paramHops);
- break;
-
- case GROUPEDAGG:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.GROUPEDAGG, paramHops);
- break;
-
- case RMEMPTY:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.RMEMPTY, paramHops);
- break;
-
- case REPLACE:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.REPLACE, paramHops);
- break;
-
- case ORDER:
- ArrayList<Hop> inputs = new ArrayList<Hop>();
- inputs.add(paramHops.get("target"));
- inputs.add(paramHops.get("by"));
- inputs.add(paramHops.get("decreasing"));
- inputs.add(paramHops.get("index.return"));
-
- currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.SORT, inputs);
-
- break;
-
- case TRANSFORM:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(),
- target.getValueType(), ParamBuiltinOp.TRANSFORM,
- paramHops);
- break;
-
- default:
-
- LOG.error(source.printErrorLocation() +
- "processParameterizedBuiltinFunctionExpression() -- Unknown operation: "
- + source.getOpCode());
-
- throw new ParseException(source.printErrorLocation() +
- "processParameterizedBuiltinFunctionExpression() -- Unknown operation: "
- + source.getOpCode());
- }
-
- setIdentifierParams(currBuiltinOp, source.getOutput());
-
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct Hops from parse tree : Process ParameterizedExpression in a
- * read/write/rand statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processDataExpression(DataExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
-
- // this expression has multiple "named" parameters
- HashMap<String, Hop> paramHops = new HashMap<String,Hop>();
-
- // -- construct hops for all input parameters
- // -- store them in hashmap so that their "name"s are maintained
- Hop pHop = null;
- for ( String paramName : source.getVarParams().keySet() ) {
- pHop = processExpression(source.getVarParam(paramName), null, hops);
- paramHops.put(paramName, pHop);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // construct hop based on opcode
- switch(source.getOpCode()) {
- case READ:
- currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTREAD, paramHops);
- ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)source.getVarParam(DataExpression.IO_FILENAME)).getValue());
- break;
-
- case WRITE:
- String name = target.getName();
- currBuiltinOp = new DataOp(
- target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTWRITE, hops.get(name), paramHops);
- //MB: commented for dynamic write
- /*Identifier ioFilename = (Identifier)source.getVarParam(DataExpression.IO_FILENAME);
- if (!(ioFilename instanceof StringIdentifier)) {
- LOG.error(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value");
- throw new ParseException(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value");
- } else {
- ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)ioFilename).getValue());
- }*/
- break;
-
- case RAND:
- // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants
- DataGenMethod method = (paramHops.get(DataExpression.RAND_MIN).getValueType()==ValueType.STRING) ?
- DataGenMethod.SINIT : DataGenMethod.RAND;
- currBuiltinOp = new DataGenOp(method, target, paramHops);
- break;
-
- case MATRIX:
- ArrayList<Hop> tmp = new ArrayList<Hop>();
- tmp.add( 0, paramHops.get(DataExpression.RAND_DATA) );
- tmp.add( 1, paramHops.get(DataExpression.RAND_ROWS) );
- tmp.add( 2, paramHops.get(DataExpression.RAND_COLS) );
- tmp.add( 3, paramHops.get(DataExpression.RAND_BY_ROW) );
- currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.RESHAPE, tmp);
- break;
-
-
- default:
- LOG.error(source.printErrorLocation() +
- "processDataExpression():: Unknown operation: "
- + source.getOpCode());
-
- throw new ParseException(source.printErrorLocation() +
- "processDataExpression():: Unknown operation: "
- + source.getOpCode());
- }
-
- //set identifier meta data (incl dimensions and blocksizes)
- setIdentifierParams(currBuiltinOp, source.getOutput());
- if( source.getOpCode()==DataExpression.DataOp.READ )
- ((DataOp)currBuiltinOp).setInputBlockSizes(target.getRowsInBlock(), target.getColumnsInBlock());
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct HOps from parse tree: process BuiltinFunction Expressions in
- * MultiAssignment Statements. For all other builtin function expressions,
- * <code>processBuiltinFunctionExpression()</code> is used.
- */
- private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList,
- HashMap<String, Hop> hops) throws ParseException {
-
- // Construct Hops for all inputs
- ArrayList<Hop> inputs = new ArrayList<Hop>();
- inputs.add( processExpression(source.getFirstExpr(), null, hops) );
- if ( source.getSecondExpr() != null )
- inputs.add( processExpression(source.getSecondExpr(), null, hops) );
- if ( source.getThirdExpr() != null )
- inputs.add( processExpression(source.getThirdExpr(), null, hops) );
-
- FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
- String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
-
- // Create an array list to hold the outputs of this lop.
- // Exact list of outputs are added based on opcode.
- ArrayList<Hop> outputs = new ArrayList<Hop>();
-
- // Construct Hop for current builtin function expression based on its type
- Hop currBuiltinOp = null;
- switch (source.getOpCode()) {
- case QR:
- case LU:
- case EIGEN:
-
- // Number of outputs = size of targetList = #of identifiers in source.getOutputs
- String[] outputNames = new String[targetList.size()];
- for ( int i=0; i < targetList.size(); i++ ) {
- outputNames[i] = ((DataIdentifier)targetList.get(i)).getName();
- Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[i]);
- outputs.add(output);
- }
-
- // Create the hop for current function call
- FunctionOp fcall = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
- currBuiltinOp = fcall;
-
- break;
-
- default:
- throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode());
- }
-
- // set properties for created hops based on outputs of source expression
- for ( int i=0; i < source.getOutputs().length; i++ ) {
- setIdentifierParams( outputs.get(i), source.getOutputs()[i]);
- outputs.get(i).setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- }
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct Hops from parse tree : Process BuiltinFunction Expression in an
- * assignment statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processBuiltinFunctionExpression(BuiltinFunctionExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
- Hop expr = processExpression(source.getFirstExpr(), null, hops);
- Hop expr2 = null;
- if (source.getSecondExpr() != null) {
- expr2 = processExpression(source.getSecondExpr(), null, hops);
- }
- Hop expr3 = null;
- if (source.getThirdExpr() != null) {
- expr3 = processExpression(source.getThirdExpr(), null, hops);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // Construct the hop based on the type of Builtin function
- switch (source.getOpCode()) {
-
- case COLSUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.Col, expr);
- break;
-
- case COLMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAX,
- Direction.Col, expr);
- break;
-
- case COLMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MIN,
- Direction.Col, expr);
- break;
-
- case COLMEAN:
- // hop to compute colSums
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.Col, expr);
- break;
-
- case ROWSUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.Row, expr);
- break;
-
- case ROWMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAX,
- Direction.Row, expr);
- break;
-
- case ROWINDEXMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAXINDEX,
- Direction.Row, expr);
- break;
-
- case ROWINDEXMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MININDEX,
- Direction.Row, expr);
- break;
-
- case ROWMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MIN,
- Direction.Row, expr);
- break;
-
- case ROWMEAN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.Row, expr);
- break;
-
- case NROW:
- // If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- // Else create a UnaryOp so that a control program instruction is generated
-
- long nRows = expr.getDim1();
- if (nRows == -1) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NROW, expr);
- }
- else {
- currBuiltinOp = new LiteralOp(nRows);
- }
- break;
-
- case NCOL:
- // If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- // Else create a UnaryOp so that a control program instruction is generated
-
- long nCols = expr.getDim2();
- if (nCols == -1) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NCOL, expr);
- }
- else {
- currBuiltinOp = new LiteralOp(nCols);
- }
- break;
- case LENGTH:
- long nRows2 = expr.getDim1();
- long nCols2 = expr.getDim2();
- /*
- * If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- * Else create a UnaryOp so that a control program instruction is generated
- */
- if ((nCols2 == -1) || (nRows2 == -1)) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.LENGTH, expr);
- }
- else {
- long lval = (nCols2 * nRows2);
- currBuiltinOp = new LiteralOp(lval);
- }
- break;
-
- case SUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.RowCol, expr);
- break;
-
- case MEAN:
- if ( expr2 == null ) {
- // example: x = mean(Y);
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.RowCol, expr);
- }
- else {
- // example: x = mean(Y,W);
- // stable weighted mean is implemented by using centralMoment with order = 0
- Hop orderHop = new LiteralOp(0);
- currBuiltinOp=new TernaryOp(target.getName(), target.getDataType(), target.getValueType(),
- Hop.OpOp3.CENTRALMOMENT, expr, expr2, orderHop);
- }
- break;
-
- case MIN:
- //construct AggUnary for min(X) but BinaryOp for min(X,Y)
- if( expr2 == null ) {
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
- AggOp.MIN, Direction.RowCol, expr);
- }
- else {
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MIN,
- expr, expr2);
- }
- break;
- case MAX:
- //construct AggUnary for max(X) but BinaryOp for max(X,Y)
- if( expr2 == null ) {
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
- AggOp.MAX, Direction.RowCol, expr);
- } else {
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MAX,
- expr, expr2);
- }
- break;
-
- case PPRED:
- String sop = ((StringIdentifier)source.getThirdExpr()).getValue();
- sop = sop.replace("\"", "");
- OpOp2 operation;
- if ( sop.equalsIgnoreCase(">=") )
- operation = OpOp2.GREATEREQUAL;
- else if ( sop.equalsIgnoreCase(">") )
- operation =
<TRUNCATED>
[04/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForMR.java
deleted file mode 100644
index 2ee9680..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForMR.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Counters.Group;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-/**
- * MR job class for submitting parfor remote MR jobs, controlling its execution and obtaining results.
- *
- *
- */
-public class RemoteDPParForMR
-{
-
- protected static final Log LOG = LogFactory.getLog(RemoteDPParForMR.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- */
- public static RemoteParForJobReturn runJob(long pfid, String itervar, String matrixvar, String program, String resultFile, MatrixObject input,
- PDataPartitionFormat dpf, OutputInfo oi, boolean tSparseCol, //config params
- boolean enableCPCaching, int numReducers, int replication, int max_retry) //opt params
- throws DMLRuntimeException
- {
- RemoteParForJobReturn ret = null;
- String jobname = "ParFor-DPEMR";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- JobConf job;
- job = new JobConf( RemoteDPParForMR.class );
- job.setJobName(jobname+pfid);
-
- //maintain dml script counters
- Statistics.incrementNoOfCompiledMRJobs();
-
- try
- {
- /////
- //configure the MR job
-
- //set arbitrary CP program blocks that will perform in the reducers
- MRJobConfiguration.setProgramBlocks(job, program);
-
- //enable/disable caching
- MRJobConfiguration.setParforCachingConfig(job, enableCPCaching);
-
- //setup input matrix
- Path path = new Path( input.getFileName() );
- long rlen = input.getNumRows();
- long clen = input.getNumColumns();
- int brlen = (int) input.getNumRowsPerBlock();
- int bclen = (int) input.getNumColumnsPerBlock();
- MRJobConfiguration.setPartitioningInfo(job, rlen, clen, brlen, bclen, InputInfo.BinaryBlockInputInfo, oi, dpf, 1, input.getFileName(), itervar, matrixvar, tSparseCol);
- job.setInputFormat(InputInfo.BinaryBlockInputInfo.inputFormatClass);
- FileInputFormat.setInputPaths(job, path);
-
- //set mapper and reducers classes
- job.setMapperClass(DataPartitionerRemoteMapper.class);
- job.setReducerClass(RemoteDPParWorkerReducer.class);
-
- //set output format
- job.setOutputFormat(SequenceFileOutputFormat.class);
-
- //set output path
- MapReduceTool.deleteFileIfExistOnHDFS(resultFile);
- FileOutputFormat.setOutputPath(job, new Path(resultFile));
-
- //set the output key, value schema
-
- //parfor partitioning outputs (intermediates)
- job.setMapOutputKeyClass(LongWritable.class);
- if( oi == OutputInfo.BinaryBlockOutputInfo )
- job.setMapOutputValueClass(PairWritableBlock.class);
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- job.setMapOutputValueClass(PairWritableCell.class);
- else
- throw new DMLRuntimeException("Unsupported intermrediate output info: "+oi);
- //parfor exec output
- job.setOutputKeyClass(LongWritable.class);
- job.setOutputValueClass(Text.class);
-
- //////
- //set optimization parameters
-
- //set the number of mappers and reducers
- job.setNumReduceTasks( numReducers );
-
- //disable automatic tasks timeouts and speculative task exec
- job.setInt("mapred.task.timeout", 0);
- job.setMapSpeculativeExecution(false);
-
- //set up preferred custom serialization framework for binary block format
- if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
- MRJobConfiguration.addBinaryBlockSerializationFramework( job );
-
- //set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
- DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
-
- //disable JVM reuse
- job.setNumTasksToExecutePerJvm( 1 ); //-1 for unlimited
-
- //set the replication factor for the results
- job.setInt("dfs.replication", replication);
-
- //set the max number of retries per map task
- //note: currently disabled to use cluster config
- //job.setInt("mapreduce.map.maxattempts", max_retry);
-
- //set unique working dir
- MRJobConfiguration.setUniqueWorkingDir(job);
-
- /////
- // execute the MR job
- RunningJob runjob = JobClient.runJob(job);
-
- // Process different counters
- Statistics.incrementNoOfExecutedMRJobs();
- Group pgroup = runjob.getCounters().getGroup(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME);
- int numTasks = (int)pgroup.getCounter( Stat.PARFOR_NUMTASKS.toString() );
- int numIters = (int)pgroup.getCounter( Stat.PARFOR_NUMITERS.toString() );
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode() ) {
- Statistics.incrementJITCompileTime( pgroup.getCounter( Stat.PARFOR_JITCOMPILE.toString() ) );
- Statistics.incrementJVMgcCount( pgroup.getCounter( Stat.PARFOR_JVMGC_COUNT.toString() ) );
- Statistics.incrementJVMgcTime( pgroup.getCounter( Stat.PARFOR_JVMGC_TIME.toString() ) );
- Group cgroup = runjob.getCounters().getGroup(CacheableData.CACHING_COUNTER_GROUP_NAME.toString());
- CacheStatistics.incrementMemHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_MEM.toString() ));
- CacheStatistics.incrementFSBuffHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_FSBUFF.toString() ));
- CacheStatistics.incrementFSHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_FS.toString() ));
- CacheStatistics.incrementHDFSHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_HDFS.toString() ));
- CacheStatistics.incrementFSBuffWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_FSBUFF.toString() ));
- CacheStatistics.incrementFSWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_FS.toString() ));
- CacheStatistics.incrementHDFSWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_HDFS.toString() ));
- CacheStatistics.incrementAcquireRTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_ACQR.toString() ));
- CacheStatistics.incrementAcquireMTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_ACQM.toString() ));
- CacheStatistics.incrementReleaseTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_RLS.toString() ));
- CacheStatistics.incrementExportTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_EXP.toString() ));
- }
-
- // read all files of result variables and prepare for return
- LocalVariableMap[] results = readResultFile(job, resultFile);
-
- ret = new RemoteParForJobReturn(runjob.isSuccessful(),
- numTasks, numIters,
- results);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- finally
- {
- // remove created files
- try
- {
- MapReduceTool.deleteFileIfExistOnHDFS(new Path(resultFile), job);
- }
- catch(IOException ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.maintainCPHeavyHitters("MR-Job_"+jobname, t1-t0);
- }
-
- return ret;
- }
-
-
- /**
- * Result file contains hierarchy of workerID-resultvar(incl filename). We deduplicate
- * on the workerID. Without JVM reuse each task refers to a unique workerID, so we
- * will not find any duplicates. With JVM reuse, however, each slot refers to a workerID,
- * and there are duplicate filenames due to partial aggregation and overwrite of fname
- * (the RemoteParWorkerMapper ensures uniqueness of those files independent of the
- * runtime implementation).
- *
- * @param job
- * @param fname
- * @return
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- public static LocalVariableMap [] readResultFile( JobConf job, String fname )
- throws DMLRuntimeException, IOException
- {
- HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>();
-
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(fname);
- LongWritable key = new LongWritable(); //workerID
- Text value = new Text(); //serialized var header (incl filename)
-
- int countAll = 0;
- for( Path lpath : MatrixReader.getSequenceFilePaths(fs, path) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(job),lpath,job);
- try
- {
- while( reader.next(key, value) )
- {
- //System.out.println("key="+key.get()+", value="+value.toString());
- if( !tmp.containsKey( key.get() ) )
- tmp.put(key.get(), new LocalVariableMap ());
- Object[] dat = ProgramConverter.parseDataObject( value.toString() );
- tmp.get( key.get() ).put((String)dat[0], (Data)dat[1]);
- countAll++;
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- LOG.debug("Num remote worker results (before deduplication): "+countAll);
- LOG.debug("Num remote worker results: "+tmp.size());
-
- //create return array
- return tmp.values().toArray(new LocalVariableMap[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSpark.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
deleted file mode 100644
index f92b50a..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.spark.Accumulator;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * TODO robustness on failures (cleanup files)
- * TODO heavy hitter maintenance
- * TODO data partitioning with binarycell
- *
- */
-public class RemoteDPParForSpark
-{
-
- protected static final Log LOG = LogFactory.getLog(RemoteDPParForSpark.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static RemoteParForJobReturn runJob(long pfid, String itervar, String matrixvar, String program, String resultFile, MatrixObject input,
- ExecutionContext ec,
- PDataPartitionFormat dpf, OutputInfo oi, boolean tSparseCol, //config params
- boolean enableCPCaching, int numReducers ) //opt params
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String jobname = "ParFor-DPESP";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- SparkExecutionContext sec = (SparkExecutionContext)ec;
- JavaSparkContext sc = sec.getSparkContext();
-
- //prepare input parameters
- MatrixDimensionsMetaData md = (MatrixDimensionsMetaData) input.getMetaData();
- MatrixCharacteristics mc = md.getMatrixCharacteristics();
- InputInfo ii = InputInfo.BinaryBlockInputInfo;
-
- //initialize accumulators for tasks/iterations
- Accumulator<Integer> aTasks = sc.accumulator(0);
- Accumulator<Integer> aIters = sc.accumulator(0);
-
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(matrixvar);
- DataPartitionerRemoteSparkMapper dpfun = new DataPartitionerRemoteSparkMapper(mc, ii, oi, dpf);
- RemoteDPParForSparkWorker efun = new RemoteDPParForSparkWorker(program, matrixvar, itervar,
- enableCPCaching, mc, tSparseCol, dpf, oi, aTasks, aIters);
- List<Tuple2<Long,String>> out =
- in.flatMapToPair(dpfun) //partition the input blocks
- .groupByKey(numReducers) //group partition blocks
- .mapPartitionsToPair( efun ) //execute parfor tasks, incl cleanup
- .collect(); //get output handles
-
- //de-serialize results
- LocalVariableMap[] results = RemoteParForUtils.getResults(out, LOG);
- int numTasks = aTasks.value(); //get accumulator value
- int numIters = aIters.value(); //get accumulator value
-
- //create output symbol table entries
- RemoteParForJobReturn ret = new RemoteParForJobReturn(true, numTasks, numIters, results);
-
- //maintain statistics
- Statistics.incrementNoOfCompiledSPInst();
- Statistics.incrementNoOfExecutedSPInst();
- if( DMLScript.STATISTICS ){
- Statistics.maintainCPHeavyHitters(jobname, System.nanoTime()-t0);
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
deleted file mode 100644
index 1338acb..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.spark.Accumulator;
-import org.apache.spark.TaskContext;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-import scala.Tuple2;
-
-/**
- *
- */
-public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapFunction<Iterator<Tuple2<Long, Iterable<Writable>>>, Long, String>
-{
- private static final long serialVersionUID = 30223759283155139L;
-
- private String _prog = null;
- private boolean _caching = true;
- private String _inputVar = null;
- private String _iterVar = null;
-
- private OutputInfo _oinfo = null;
- private int _rlen = -1;
- private int _clen = -1;
- private int _brlen = -1;
- private int _bclen = -1;
- private boolean _tSparseCol = false;
- private PDataPartitionFormat _dpf = null;
-
- private Accumulator<Integer> _aTasks = null;
- private Accumulator<Integer> _aIters = null;
-
- public RemoteDPParForSparkWorker(String program, String inputVar, String iterVar, boolean cpCaching, MatrixCharacteristics mc, boolean tSparseCol, PDataPartitionFormat dpf, OutputInfo oinfo, Accumulator<Integer> atasks, Accumulator<Integer> aiters)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //keep inputs (unfortunately, spark does not expose task ids and it would be implementation-dependent
- //when this constructor is actually called; hence, we do lazy initialization on task execution)
- _prog = program;
- _caching = cpCaching;
- _inputVar = inputVar;
- _iterVar = iterVar;
- _oinfo = oinfo;
-
- //setup spark accumulators
- _aTasks = atasks;
- _aIters = aiters;
-
- //setup matrixblock partition and meta data
- _rlen = (int)mc.getRows();
- _clen = (int)mc.getCols();
- _brlen = mc.getRowsPerBlock();
- _bclen = mc.getColsPerBlock();
- _tSparseCol = tSparseCol;
- _dpf = dpf;
- switch( _dpf ) { //create matrix partition for reuse
- case ROW_WISE: _rlen = 1; break;
- case COLUMN_WISE: _clen = 1; break;
- default: throw new RuntimeException("Partition format not yet supported in fused partition-execute: "+dpf);
- }
- }
-
- @Override
- public Iterable<Tuple2<Long, String>> call(Iterator<Tuple2<Long, Iterable<Writable>>> arg0)
- throws Exception
- {
- ArrayList<Tuple2<Long,String>> ret = new ArrayList<Tuple2<Long,String>>();
-
- //lazy parworker initialization
- configureWorker( TaskContext.get().taskAttemptId() ); //requires Spark 1.3
-
- //process all matrix partitions of this data partition
- while( arg0.hasNext() )
- {
- Tuple2<Long,Iterable<Writable>> larg = arg0.next();
-
- //collect input partition (check via equals because oinfo deserialized instance)
- MatrixBlock partition = null;
- if( _oinfo.equals(OutputInfo.BinaryBlockOutputInfo) )
- partition = collectBinaryBlock( larg._2() );
- else
- partition = collectBinaryCellInput( larg._2() );
-
- //update in-memory matrix partition
- MatrixObject mo = (MatrixObject)_ec.getVariable( _inputVar );
- mo.setInMemoryPartition( partition );
-
- //create tasks for input data
- Task lTask = new Task(TaskType.SET);
- lTask.addIteration( new IntObject(_iterVar, larg._1()) );
-
- //execute program
- long numIter = getExecutedIterations();
- super.executeTask( lTask );
-
- //maintain accumulators
- _aTasks.add( 1 );
- _aIters.add( (int)(getExecutedIterations()-numIter) );
-
- //write output if required (matrix indexed write)
- //note: this copy is necessary for environments without spark libraries
- ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars );
- for( String val : tmp )
- ret.add(new Tuple2<Long,String>(_workerID, val));
- }
-
- return ret;
- }
-
- /**
- *
- * @param ID
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void configureWorker( long ID )
- throws DMLRuntimeException, DMLUnsupportedOperationException, IOException
- {
- _workerID = ID;
-
- //parse and setup parfor body program
- ParForBody body = ProgramConverter.parseParForBody(_prog, (int)_workerID);
- _childBlocks = body.getChildBlocks();
- _ec = body.getEc();
- _resultVars = body.getResultVarNames();
- _numTasks = 0;
- _numIters = 0;
-
- //init local cache manager
- if( !CacheableData.isCachingActive() ) {
- String uuid = IDHandler.createDistributedUniqueID();
- LocalFileUtils.createWorkingDirectoryWithUUID( uuid );
- CacheableData.initCaching( uuid ); //incl activation, cache dir creation (each map task gets its own dir for simplified cleanup)
- }
- if( !CacheableData.cacheEvictionLocalFilePrefix.contains("_") ){ //account for local mode
- CacheableData.cacheEvictionLocalFilePrefix = CacheableData.cacheEvictionLocalFilePrefix +"_" + _workerID;
- }
-
- //ensure that resultvar files are not removed
- super.pinResultVariables();
-
- //enable/disable caching (if required)
- if( !_caching )
- CacheableData.disableCaching();
- }
-
- /**
- * Collects a matrixblock partition from a given input iterator over
- * binary blocks.
- *
- * Note it reuses the instance attribute _partition - multiple calls
- * will overwrite the result.
- *
- * @param valueList
- * @return
- * @throws IOException
- */
- private MatrixBlock collectBinaryBlock( Iterable<Writable> valueList )
- throws IOException
- {
- MatrixBlock partition = null;
-
- try
- {
- //reset reuse block, keep configured representation
- if( _tSparseCol )
- partition = new MatrixBlock(_clen, _rlen, true);
- else
- partition = new MatrixBlock(_rlen, _clen, false);
-
- for( Writable val : valueList )
- {
- PairWritableBlock pairValue = (PairWritableBlock) val;
- int row_offset = (int)(pairValue.indexes.getRowIndex()-1)*_brlen;
- int col_offset = (int)(pairValue.indexes.getColumnIndex()-1)*_bclen;
- MatrixBlock block = pairValue.block;
- if( !partition.isInSparseFormat() ) //DENSE
- {
- partition.copy( row_offset, row_offset+block.getNumRows()-1,
- col_offset, col_offset+block.getNumColumns()-1,
- pairValue.block, false );
- }
- else //SPARSE
- {
- partition.appendToSparse(pairValue.block, row_offset, col_offset);
- }
- }
-
- //final partition cleanup
- cleanupCollectedMatrixPartition( partition, partition.isInSparseFormat() );
- }
- catch(DMLRuntimeException ex)
- {
- throw new IOException(ex);
- }
-
- return partition;
- }
-
-
- /**
- * Collects a matrixblock partition from a given input iterator over
- * binary cells.
- *
- * Note it reuses the instance attribute _partition - multiple calls
- * will overwrite the result.
- *
- * @param valueList
- * @return
- * @throws IOException
- */
- private MatrixBlock collectBinaryCellInput( Iterable<Writable> valueList )
- throws IOException
- {
- MatrixBlock partition = null;
-
- //reset reuse block, keep configured representation
- if( _tSparseCol )
- partition = new MatrixBlock(_clen, _rlen, true);
- else
- partition = new MatrixBlock(_rlen, _clen, false);
-
- switch( _dpf )
- {
- case ROW_WISE:
- while( valueList.iterator().hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.iterator().next();
- if( pairValue.indexes.getColumnIndex()<0 )
- continue; //cells used to ensure empty partitions
- partition.quickSetValue(0, (int)pairValue.indexes.getColumnIndex()-1, pairValue.cell.getValue());
- }
- break;
- case COLUMN_WISE:
- while( valueList.iterator().hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.iterator().next();
- if( pairValue.indexes.getRowIndex()<0 )
- continue; //cells used to ensure empty partitions
- if( _tSparseCol )
- partition.appendValue(0,(int)pairValue.indexes.getRowIndex()-1, pairValue.cell.getValue());
- else
- partition.quickSetValue((int)pairValue.indexes.getRowIndex()-1, 0, pairValue.cell.getValue());
- }
- break;
- default:
- throw new IOException("Partition format not yet supported in fused partition-execute: "+_dpf);
- }
-
- //final partition cleanup
- cleanupCollectedMatrixPartition(partition, _tSparseCol);
-
- return partition;
- }
-
- /**
- *
- * @param sort
- * @throws IOException
- */
- private void cleanupCollectedMatrixPartition(MatrixBlock partition, boolean sort)
- throws IOException
- {
- //sort sparse row contents if required
- if( partition.isInSparseFormat() && sort )
- partition.sortSparseRows();
-
- //ensure right number of nnz
- if( !partition.isInSparseFormat() )
- partition.recomputeNonZeros();
-
- //exam and switch dense/sparse representation
- try {
- partition.examSparsity();
- }
- catch(Exception ex){
- throw new IOException(ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
deleted file mode 100644
index 6aec002..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- *
- */
-public class RemoteDPParWorkerReducer extends ParWorker
- implements Reducer<LongWritable, Writable, Writable, Writable>
-{
-
- //MR data partitioning attributes
- private String _inputVar = null;
- private String _iterVar = null;
- private PDataPartitionFormat _dpf = null;
- private OutputInfo _info = null;
- private int _rlen = -1;
- private int _clen = -1;
- private int _brlen = -1;
- private int _bclen = -1;
-
- //reuse matrix partition
- private MatrixBlock _partition = null;
- private boolean _tSparseCol = false;
-
- //MR ParWorker attributes
- protected String _stringID = null;
- protected HashMap<String, String> _rvarFnames = null;
-
- //cached collector/reporter
- protected OutputCollector<Writable, Writable> _out = null;
- protected Reporter _report = null;
-
- /**
- *
- */
- public RemoteDPParWorkerReducer()
- {
-
- }
-
- @Override
- public void reduce(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- //cache collector/reporter (for write in close)
- _out = out;
- _report = reporter;
-
- //collect input partition
- if( _info == OutputInfo.BinaryBlockOutputInfo )
- _partition = collectBinaryBlock( valueList );
- else
- _partition = collectBinaryCellInput( valueList );
-
- //update in-memory matrix partition
- MatrixObject mo = (MatrixObject)_ec.getVariable( _inputVar );
- mo.setInMemoryPartition( _partition );
-
- //execute program
- LOG.trace("execute RemoteDPParWorkerReducer "+_stringID+" ("+_workerID+")");
- try {
- //create tasks for input data
- Task lTask = new Task(TaskType.SET);
- lTask.addIteration( new IntObject(_iterVar,key.get()) );
-
- //execute program
- executeTask( lTask );
- }
- catch(Exception ex)
- {
- throw new IOException("ParFOR: Failed to execute task.",ex);
- }
-
- //statistic maintenance (after final export)
- RemoteParForUtils.incrementParForMRCounters(_report, 1, 1);
- }
-
- /**
- *
- */
- @Override
- public void configure(JobConf job)
- {
- //Step 1: configure data partitioning information
- _rlen = (int)MRJobConfiguration.getPartitioningNumRows( job );
- _clen = (int)MRJobConfiguration.getPartitioningNumCols( job );
- _brlen = MRJobConfiguration.getPartitioningBlockNumRows( job );
- _bclen = MRJobConfiguration.getPartitioningBlockNumCols( job );
- _iterVar = MRJobConfiguration.getPartitioningItervar( job );
- _inputVar = MRJobConfiguration.getPartitioningMatrixvar( job );
- _dpf = MRJobConfiguration.getPartitioningFormat( job );
- switch( _dpf ) { //create matrix partition for reuse
- case ROW_WISE: _rlen = 1; break;
- case COLUMN_WISE: _clen = 1; break;
- default: throw new RuntimeException("Partition format not yet supported in fused partition-execute: "+_dpf);
- }
- _info = MRJobConfiguration.getPartitioningOutputInfo( job );
- _tSparseCol = MRJobConfiguration.getPartitioningTransposedCol( job );
- if( _tSparseCol )
- _partition = new MatrixBlock((int)_clen, _rlen, true);
- else
- _partition = new MatrixBlock((int)_rlen, _clen, false);
-
- //Step 1: configure parworker
- String taskID = job.get("mapred.tip.id");
- LOG.trace("configure RemoteDPParWorkerReducer "+taskID);
-
- try
- {
- _stringID = taskID;
- _workerID = IDHandler.extractIntID(_stringID); //int task ID
-
- //use the given job configuration as source for all new job confs
- //NOTE: this is required because on HDP 2.3, the classpath of mr tasks contained hadoop-common.jar
- //which includes a core-default.xml configuration which hides the actual default cluster configuration
- //in the context of mr jobs (for example this config points to local fs instead of hdfs by default).
- if( !InfrastructureAnalyzer.isLocalMode(job) ) {
- ConfigurationManager.setCachedJobConf(job);
- }
-
- //create local runtime program
- String in = MRJobConfiguration.getProgramBlocks(job);
- ParForBody body = ProgramConverter.parseParForBody(in, (int)_workerID);
- _childBlocks = body.getChildBlocks();
- _ec = body.getEc();
- _resultVars = body.getResultVarNames();
-
- //init local cache manager
- if( !CacheableData.isCachingActive() ) {
- String uuid = IDHandler.createDistributedUniqueID();
- LocalFileUtils.createWorkingDirectoryWithUUID( uuid );
- CacheableData.initCaching( uuid ); //incl activation, cache dir creation (each map task gets its own dir for simplified cleanup)
- }
- if( !CacheableData.cacheEvictionLocalFilePrefix.contains("_") ){ //account for local mode
- CacheableData.cacheEvictionLocalFilePrefix = CacheableData.cacheEvictionLocalFilePrefix +"_" + _workerID;
- }
-
- //ensure that resultvar files are not removed
- super.pinResultVariables();
-
- //enable/disable caching (if required)
- boolean cpCaching = MRJobConfiguration.getParforCachingConfig( job );
- if( !cpCaching )
- CacheableData.disableCaching();
-
- _numTasks = 0;
- _numIters = 0;
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
-
- //disable parfor stat monitoring, reporting execution times via counters not useful
- StatisticMonitor.disableStatMonitoring();
-
- //always reset stats because counters per map task (for case of JVM reuse)
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- {
- CacheStatistics.reset();
- Statistics.reset();
- }
- }
-
- /**
- *
- */
- @Override
- public void close()
- throws IOException
- {
- try
- {
- //write output if required (matrix indexed write)
- RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars, _out );
-
- //statistic maintenance (after final export)
- RemoteParForUtils.incrementParForMRCounters(_report, 0, 0);
-
- //print heaver hitter per task
- JobConf job = ConfigurationManager.getCachedJobConf();
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- LOG.info("\nSystemML Statistics:\nHeavy hitter instructions (name, time, count):\n" + Statistics.getHeavyHitters(10));
- }
- catch(Exception ex)
- {
- throw new IOException( ex );
- }
-
- //cleanup cache and local tmp dir
- RemoteParForUtils.cleanupWorkingDirectories();
-
- //ensure caching is not disabled for CP in local mode
- CacheableData.enableCaching();
- }
-
- /**
- * Collects a matrixblock partition from a given input iterator over
- * binary blocks.
- *
- * Note it reuses the instance attribute _partition - multiple calls
- * will overwrite the result.
- *
- * @param valueList
- * @return
- * @throws IOException
- */
- private MatrixBlock collectBinaryBlock( Iterator<Writable> valueList )
- throws IOException
- {
- try
- {
- //reset reuse block, keep configured representation
- _partition.reset(_rlen, _clen);
-
- while( valueList.hasNext() )
- {
- PairWritableBlock pairValue = (PairWritableBlock)valueList.next();
- int row_offset = (int)(pairValue.indexes.getRowIndex()-1)*_brlen;
- int col_offset = (int)(pairValue.indexes.getColumnIndex()-1)*_bclen;
- MatrixBlock block = pairValue.block;
- if( !_partition.isInSparseFormat() ) //DENSE
- {
- _partition.copy( row_offset, row_offset+block.getNumRows()-1,
- col_offset, col_offset+block.getNumColumns()-1,
- pairValue.block, false );
- }
- else //SPARSE
- {
- _partition.appendToSparse(pairValue.block, row_offset, col_offset);
- }
- }
-
- //final partition cleanup
- cleanupCollectedMatrixPartition( _partition.isInSparseFormat() );
- }
- catch(DMLRuntimeException ex)
- {
- throw new IOException(ex);
- }
-
- return _partition;
- }
-
-
- /**
- * Collects a matrixblock partition from a given input iterator over
- * binary cells.
- *
- * Note it reuses the instance attribute _partition - multiple calls
- * will overwrite the result.
- *
- * @param valueList
- * @return
- * @throws IOException
- */
- private MatrixBlock collectBinaryCellInput( Iterator<Writable> valueList )
- throws IOException
- {
- //reset reuse block, keep configured representation
- if( _tSparseCol )
- _partition.reset(_clen, _rlen);
- else
- _partition.reset(_rlen, _clen);
-
- switch( _dpf )
- {
- case ROW_WISE:
- while( valueList.hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.next();
- if( pairValue.indexes.getColumnIndex()<0 )
- continue; //cells used to ensure empty partitions
- _partition.quickSetValue(0, (int)pairValue.indexes.getColumnIndex()-1, pairValue.cell.getValue());
- }
- break;
- case COLUMN_WISE:
- while( valueList.hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.next();
- if( pairValue.indexes.getRowIndex()<0 )
- continue; //cells used to ensure empty partitions
- if( _tSparseCol )
- _partition.appendValue(0,(int)pairValue.indexes.getRowIndex()-1, pairValue.cell.getValue());
- else
- _partition.quickSetValue((int)pairValue.indexes.getRowIndex()-1, 0, pairValue.cell.getValue());
- }
- break;
- default:
- throw new IOException("Partition format not yet supported in fused partition-execute: "+_dpf);
- }
-
- //final partition cleanup
- cleanupCollectedMatrixPartition(_tSparseCol);
-
- return _partition;
- }
-
- /**
- *
- * @param sort
- * @throws IOException
- */
- private void cleanupCollectedMatrixPartition(boolean sort)
- throws IOException
- {
- //sort sparse row contents if required
- if( _partition.isInSparseFormat() && sort )
- _partition.sortSparseRows();
-
- //ensure right number of nnz
- if( !_partition.isInSparseFormat() )
- _partition.recomputeNonZeros();
-
- //exam and switch dense/sparse representation
- try {
- _partition.examSparsity();
- }
- catch(Exception ex){
- throw new IOException(ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
deleted file mode 100644
index 580a4e3..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.lib.NLineInputFormat;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-
-/**
- * Wrapper for arbitrary input split in order to attach our co-location information.
- *
- */
-public class RemoteParForColocatedFileSplit extends FileSplit
-{
-
- private String _fname = null;
- private int _blen = 1;
-
- /**
- * Required because hadoop explicitly accesses this private constructor
- * via reflection (since private not inherited from FileSplit).
- */
- @SuppressWarnings("unused")
- private RemoteParForColocatedFileSplit()
- {
- super( null, -1, -1, new String[]{} );
- }
-
- public RemoteParForColocatedFileSplit( FileSplit split, String fname, int blen )
- throws IOException
- {
- super( split.getPath(), split.getStart(), split.getLength(), split.getLocations() );
-
- _fname = fname;
- _blen = blen;
- }
-
- /**
- * Get the list of hostnames where the input split is located.
- */
- @Override
- public String[] getLocations() throws IOException
- {
- //Timing time = new Timing();
- //time.start();
-
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
-
- //read task string
- LongWritable key = new LongWritable();
- Text value = new Text();
- RecordReader<LongWritable,Text> reader = new NLineInputFormat().getRecordReader(this, job, Reporter.NULL);
- reader.next(key, value);
- reader.close();
-
- //parse task
- Task t = Task.parseCompactString( value.toString() );
-
- //get all locations
- HashMap<String, Integer> hosts = new HashMap<String,Integer>();
-
- if( t.getType() == TaskType.SET )
- {
- for( IntObject val : t.getIterations() )
- {
- String fname = _fname+"/"+String.valueOf(((val.getLongValue()-1)/_blen+1));
- FileStatus status = fs.getFileStatus(new Path(fname));
- BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
- for( BlockLocation bl : tmp1 )
- countHosts(hosts, bl.getHosts());
- }
- }
- else //TaskType.RANGE
- {
- //since this is a serial process, we use just the first iteration
- //as a heuristic for location information
- long lFrom = t.getIterations().get(0).getLongValue();
- long lTo = t.getIterations().get(1).getLongValue();
- for( long li : new long[]{lFrom,lTo} )
- {
- String fname = _fname+"/"+String.valueOf( ((li-1)/_blen+1) );
- FileStatus status = fs.getFileStatus(new Path(fname));
- BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
- for( BlockLocation bl : tmp1 )
- countHosts(hosts, bl.getHosts());
- }
-
- /*
- int lFrom = t.getIterations().get(0).getIntValue();
- int lTo = t.getIterations().get(1).getIntValue();
- int lIncr = t.getIterations().get(2).getIntValue();
- for( int i=lFrom; i<=lTo; i+=lIncr )
- {
- String fname = _fname+"/"+String.valueOf( ((i-_offset)/_blen+_offset) );
- FileSystem fs = FileSystem.get(job);
- FileStatus status = fs.getFileStatus(new Path(fname));
- BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
- for( BlockLocation bl : tmp1 )
- countHosts(hosts, bl.getHosts());
- }*/
- }
-
- //System.out.println("Get locations "+time.stop()+"");
-
- //majority consensus on top host
- return getTopHosts(hosts);
- }
-
-
- /**
- *
- * @param hosts
- * @param names
- */
- private void countHosts( HashMap<String,Integer> hosts, String[] names )
- {
- for( String name : names )
- {
- Integer tmp = hosts.get(name);
- if( tmp != null )
- hosts.put(name, tmp+1);
- else
- hosts.put(name, 1);
- }
- }
-
- /**
- *
- * @param hosts
- * @return
- */
- private String[] getTopHosts( HashMap<String,Integer> hosts )
- {
- int max = Integer.MIN_VALUE;
- HashSet<String> maxName = new HashSet<String>();
-
- for( Entry<String,Integer> e : hosts.entrySet() )
- if( e.getValue() > max ) {
- maxName.clear();
- max = e.getValue();
- maxName.add(e.getKey());
- }
- else if( e.getValue() == max )
- maxName.add(e.getKey());
-
- //System.out.println("HOSTS: "+ProgramConverter.serializeStringHashSet(maxName));
- return maxName.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
deleted file mode 100644
index 8243b6d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.lib.NLineInputFormat;
-
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-
-/**
- * Specific extension of NLineInputFormat in order to ensure data colocation
- * for partitioned matrices although those matrices are not directly passed to the
- * MR job as an input.
- *
- */
-public class RemoteParForColocatedNLineInputFormat extends NLineInputFormat
-{
-
- @Override
- public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException
- {
- InputSplit[] tmp = super.getSplits(job, numSplits);
-
- //get partitioning information
- PDataPartitionFormat dpf = MRJobConfiguration.getPartitioningFormat(job);
- int blen = -1;
- switch( dpf ) {
- case ROW_WISE: blen = 1; break;
- case ROW_BLOCK_WISE: blen = MRJobConfiguration.getPartitioningBlockNumRows(job); break;
- case COLUMN_WISE: blen = 1; break;
- case COLUMN_BLOCK_WISE: blen = MRJobConfiguration.getPartitioningBlockNumCols(job); break;
- default:
- //do nothing
- }
- String fname = MRJobConfiguration.getPartitioningFilename(job);
-
- //create wrapper splits
- InputSplit[] ret = new InputSplit[ tmp.length ];
- for( int i=0; i<tmp.length; i++ )
- {
- //check for robustness of subsequent cast
- if( tmp[i] instanceof FileSplit )
- ret[i] = new RemoteParForColocatedFileSplit( (FileSplit) tmp[i], fname, blen );
- else
- ret[i] = tmp[i];
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForJobReturn.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
deleted file mode 100644
index e052f41..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-
-/**
- * Wrapper for job return of ParFor REMOTE for transferring statistics and result symbol table.
- *
- */
-public class RemoteParForJobReturn
-{
-
-
- private boolean _successful = true;
- private int _numTasks = -1;
- private int _numIters = -1;
- private LocalVariableMap [] _variables = null;
-
- public RemoteParForJobReturn( boolean successful, int numTasks, int numIters, LocalVariableMap [] variables )
- {
- _successful = successful;
- _numTasks = numTasks;
- _numIters = numIters;
- _variables = variables;
- }
-
- /**
- *
- * @return
- */
- public boolean isSuccessful()
- {
- return _successful;
- }
-
- /**
- *
- * @return
- */
- public int getNumExecutedTasks()
- {
- return _numTasks;
- }
-
- /**
- *
- * @return
- */
- public int getNumExecutedIterations()
- {
- return _numIters;
- }
-
- /**
- *
- * @return
- */
- public LocalVariableMap [] getVariables()
- {
- return _variables;
- }
-
- /**
- *
- * @param variables
- */
- public void setVariables (LocalVariableMap [] variables)
- {
- _variables = variables;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForMR.java
deleted file mode 100644
index b74ae1f..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForMR.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Counters.Group;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.lib.NLineInputFormat;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-/**
- * MR job class for submitting parfor remote MR jobs, controlling its execution and obtaining results.
- *
- *
- */
-public class RemoteParForMR
-{
-
- protected static final Log LOG = LogFactory.getLog(RemoteParForMR.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param _enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- */
- public static RemoteParForJobReturn runJob(long pfid, String program, String taskFile, String resultFile, MatrixObject colocatedDPMatrixObj, //inputs
- boolean enableCPCaching, int numMappers, int replication, int max_retry, long minMem, boolean jvmReuse) //opt params
- throws DMLRuntimeException
- {
- RemoteParForJobReturn ret = null;
- String jobname = "ParFor-EMR";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- JobConf job;
- job = new JobConf( RemoteParForMR.class );
- job.setJobName(jobname+pfid);
-
- //maintain dml script counters
- Statistics.incrementNoOfCompiledMRJobs();
-
- try
- {
- /////
- //configure the MR job
-
- //set arbitrary CP program blocks that will perform in the mapper
- MRJobConfiguration.setProgramBlocks(job, program);
-
- //enable/disable caching
- MRJobConfiguration.setParforCachingConfig(job, enableCPCaching);
-
- //set mappers, reducers, combiners
- job.setMapperClass(RemoteParWorkerMapper.class); //map-only
-
- //set input format (one split per row, NLineInputFormat default N=1)
- if( ParForProgramBlock.ALLOW_DATA_COLOCATION && colocatedDPMatrixObj != null )
- {
- job.setInputFormat(RemoteParForColocatedNLineInputFormat.class);
- MRJobConfiguration.setPartitioningFormat(job, colocatedDPMatrixObj.getPartitionFormat());
- MatrixCharacteristics mc = colocatedDPMatrixObj.getMatrixCharacteristics();
- MRJobConfiguration.setPartitioningBlockNumRows(job, mc.getRowsPerBlock());
- MRJobConfiguration.setPartitioningBlockNumCols(job, mc.getColsPerBlock());
- MRJobConfiguration.setPartitioningFilename(job, colocatedDPMatrixObj.getFileName());
- }
- else //default case
- {
- job.setInputFormat(NLineInputFormat.class);
- }
-
- //set the input path and output path
- FileInputFormat.setInputPaths(job, new Path(taskFile));
-
- //set output format
- job.setOutputFormat(SequenceFileOutputFormat.class);
-
- //set output path
- MapReduceTool.deleteFileIfExistOnHDFS(resultFile);
- FileOutputFormat.setOutputPath(job, new Path(resultFile));
-
- //set the output key, value schema
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(Text.class);
- job.setOutputKeyClass(LongWritable.class);
- job.setOutputValueClass(Text.class);
-
- //////
- //set optimization parameters
-
- //set the number of mappers and reducers
- job.setNumMapTasks(numMappers); //numMappers
- job.setNumReduceTasks( 0 );
- //job.setInt("mapred.map.tasks.maximum", 1); //system property
- //job.setInt("mapred.tasktracker.tasks.maximum",1); //system property
- //job.setInt("mapred.jobtracker.maxtasks.per.job",1); //system property
-
- //use FLEX scheduler configuration properties
- if( ParForProgramBlock.USE_FLEX_SCHEDULER_CONF )
- {
- job.setInt("flex.priority",0); //highest
-
- job.setInt("flex.map.min", 0);
- job.setInt("flex.map.max", numMappers);
- job.setInt("flex.reduce.min", 0);
- job.setInt("flex.reduce.max", numMappers);
- }
-
- //set jvm memory size (if require)
- String memKey = "mapred.child.java.opts";
- if( minMem > 0 && minMem > InfrastructureAnalyzer.extractMaxMemoryOpt(job.get(memKey)) )
- {
- InfrastructureAnalyzer.setMaxMemoryOpt(job, memKey, minMem);
- LOG.warn("Forcing '"+memKey+"' to -Xmx"+minMem/(1024*1024)+"M." );
- }
-
- //disable automatic tasks timeouts and speculative task exec
- job.setInt("mapred.task.timeout", 0);
- job.setMapSpeculativeExecution(false);
-
- //set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
- DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
-
- //enables the reuse of JVMs (multiple tasks per MR task)
- if( jvmReuse )
- job.setNumTasksToExecutePerJvm(-1); //unlimited
-
- //set sort io buffer (reduce unnecessary large io buffer, guaranteed memory consumption)
- job.setInt("io.sort.mb", 8); //8MB
-
- //set the replication factor for the results
- job.setInt("dfs.replication", replication);
-
- //set the max number of retries per map task
- // disabled job-level configuration to respect cluster configuration
- // note: this refers to hadoop2, hence it never had effect on mr1
- //job.setInt("mapreduce.map.maxattempts", max_retry);
-
- //set unique working dir
- MRJobConfiguration.setUniqueWorkingDir(job);
-
-
- /////
- // execute the MR job
- RunningJob runjob = JobClient.runJob(job);
-
- // Process different counters
- Statistics.incrementNoOfExecutedMRJobs();
- Group pgroup = runjob.getCounters().getGroup(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME);
- int numTasks = (int)pgroup.getCounter( Stat.PARFOR_NUMTASKS.toString() );
- int numIters = (int)pgroup.getCounter( Stat.PARFOR_NUMITERS.toString() );
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode() ) {
- Statistics.incrementJITCompileTime( pgroup.getCounter( Stat.PARFOR_JITCOMPILE.toString() ) );
- Statistics.incrementJVMgcCount( pgroup.getCounter( Stat.PARFOR_JVMGC_COUNT.toString() ) );
- Statistics.incrementJVMgcTime( pgroup.getCounter( Stat.PARFOR_JVMGC_TIME.toString() ) );
- Group cgroup = runjob.getCounters().getGroup(CacheableData.CACHING_COUNTER_GROUP_NAME.toString());
- CacheStatistics.incrementMemHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_MEM.toString() ));
- CacheStatistics.incrementFSBuffHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_FSBUFF.toString() ));
- CacheStatistics.incrementFSHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_FS.toString() ));
- CacheStatistics.incrementHDFSHits((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_HITS_HDFS.toString() ));
- CacheStatistics.incrementFSBuffWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_FSBUFF.toString() ));
- CacheStatistics.incrementFSWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_FS.toString() ));
- CacheStatistics.incrementHDFSWrites((int)cgroup.getCounter( CacheStatistics.Stat.CACHE_WRITES_HDFS.toString() ));
- CacheStatistics.incrementAcquireRTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_ACQR.toString() ));
- CacheStatistics.incrementAcquireMTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_ACQM.toString() ));
- CacheStatistics.incrementReleaseTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_RLS.toString() ));
- CacheStatistics.incrementExportTime(cgroup.getCounter( CacheStatistics.Stat.CACHE_TIME_EXP.toString() ));
- }
-
- // read all files of result variables and prepare for return
- LocalVariableMap[] results = readResultFile(job, resultFile);
-
- ret = new RemoteParForJobReturn(runjob.isSuccessful(),
- numTasks, numIters,
- results);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- finally
- {
- // remove created files
- try
- {
- MapReduceTool.deleteFileIfExistOnHDFS(new Path(taskFile), job);
- MapReduceTool.deleteFileIfExistOnHDFS(new Path(resultFile), job);
- }
- catch(IOException ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.maintainCPHeavyHitters("MR-Job_"+jobname, t1-t0);
- }
-
- return ret;
- }
-
-
- /**
- * Result file contains hierarchy of workerID-resultvar(incl filename). We deduplicate
- * on the workerID. Without JVM reuse each task refers to a unique workerID, so we
- * will not find any duplicates. With JVM reuse, however, each slot refers to a workerID,
- * and there are duplicate filenames due to partial aggregation and overwrite of fname
- * (the RemoteParWorkerMapper ensures uniqueness of those files independent of the
- * runtime implementation).
- *
- * @param job
- * @param fname
- * @return
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- public static LocalVariableMap [] readResultFile( JobConf job, String fname )
- throws DMLRuntimeException, IOException
- {
- HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>();
-
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(fname);
- LongWritable key = new LongWritable(); //workerID
- Text value = new Text(); //serialized var header (incl filename)
-
- int countAll = 0;
- for( Path lpath : MatrixReader.getSequenceFilePaths(fs, path) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(job),lpath,job);
- try
- {
- while( reader.next(key, value) )
- {
- //System.out.println("key="+key.get()+", value="+value.toString());
- if( !tmp.containsKey( key.get() ) )
- tmp.put(key.get(), new LocalVariableMap ());
- Object[] dat = ProgramConverter.parseDataObject( value.toString() );
- tmp.get( key.get() ).put((String)dat[0], (Data)dat[1]);
- countAll++;
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- LOG.debug("Num remote worker results (before deduplication): "+countAll);
- LOG.debug("Num remote worker results: "+tmp.size());
-
- //create return array
- return tmp.values().toArray(new LocalVariableMap[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSpark.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSpark.java
deleted file mode 100644
index 255ad15..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSpark.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.spark.Accumulator;
-import org.apache.spark.api.java.JavaSparkContext;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * This class serves two purposes: (1) isolating Spark imports to enable running in
- * environments where no Spark libraries are available, and (2) to follow the same
- * structure as the parfor remote_mr job submission.
- *
- * NOTE: currently, we still exchange inputs and outputs via hdfs (this covers the general case
- * if data already resides in HDFS, in-memory data, and partitioned inputs; also, it allows for
- * pre-aggregation by overwriting partial task results with pre-paggregated results from subsequent
- * iterations)
- *
- * TODO broadcast variables if possible
- * TODO reducebykey on variable names
- */
-public class RemoteParForSpark
-{
-
- protected static final Log LOG = LogFactory.getLog(RemoteParForSpark.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param tasks
- * @param ec
- * @param enableCPCaching
- * @param numMappers
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static RemoteParForJobReturn runJob(long pfid, String program, List<Task> tasks, ExecutionContext ec,
- boolean cpCaching, int numMappers)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- String jobname = "ParFor-ESP";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- SparkExecutionContext sec = (SparkExecutionContext)ec;
- JavaSparkContext sc = sec.getSparkContext();
-
- //initialize accumulators for tasks/iterations
- Accumulator<Integer> aTasks = sc.accumulator(0);
- Accumulator<Integer> aIters = sc.accumulator(0);
-
- //run remote_spark parfor job
- //(w/o lazy evaluation to fit existing parfor framework, e.g., result merge)
- RemoteParForSparkWorker func = new RemoteParForSparkWorker(program, cpCaching, aTasks, aIters);
- List<Tuple2<Long,String>> out =
- sc.parallelize( tasks, numMappers ) //create rdd of parfor tasks
- .flatMapToPair( func ) //execute parfor tasks
- .collect(); //get output handles
-
- //de-serialize results
- LocalVariableMap[] results = RemoteParForUtils.getResults(out, LOG);
- int numTasks = aTasks.value(); //get accumulator value
- int numIters = aIters.value(); //get accumulator value
-
- //create output symbol table entries
- RemoteParForJobReturn ret = new RemoteParForJobReturn(true, numTasks, numIters, results);
-
- //maintain statistics
- Statistics.incrementNoOfCompiledSPInst();
- Statistics.incrementNoOfExecutedSPInst();
- if( DMLScript.STATISTICS ){
- Statistics.maintainCPHeavyHitters(jobname, System.nanoTime()-t0);
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
deleted file mode 100644
index bfa7e96..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.spark.Accumulator;
-import org.apache.spark.TaskContext;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-import scala.Tuple2;
-
-/**
- *
- *
- */
-public class RemoteParForSparkWorker extends ParWorker implements PairFlatMapFunction<Task, Long, String>
-{
-
- private static final long serialVersionUID = -3254950138084272296L;
-
- private boolean _initialized = false;
- private String _prog = null;
- private boolean _caching = true;
-
- private Accumulator<Integer> _aTasks = null;
- private Accumulator<Integer> _aIters = null;
-
- public RemoteParForSparkWorker(String program, boolean cpCaching, Accumulator<Integer> atasks, Accumulator<Integer> aiters)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //keep inputs (unfortunately, spark does not expose task ids and it would be implementation-dependent
- //when this constructor is actually called; hence, we do lazy initialization on task execution)
- _initialized = false;
- _prog = program;
- _caching = cpCaching;
-
- //setup spark accumulators
- _aTasks = atasks;
- _aIters = aiters;
- }
-
- @Override
- public Iterable<Tuple2<Long, String>> call(Task arg0)
- throws Exception
- {
- //lazy parworker initialization
- if( !_initialized )
- configureWorker( TaskContext.get().taskAttemptId() ); //requires Spark 1.3
-
- //execute a single task
- long numIter = getExecutedIterations();
- super.executeTask( arg0 );
-
- //maintain accumulators
- _aTasks.add( 1 );
- _aIters.add( (int)(getExecutedIterations()-numIter) );
-
- //write output if required (matrix indexed write)
- //note: this copy is necessary for environments without spark libraries
- ArrayList<Tuple2<Long,String>> ret = new ArrayList<Tuple2<Long,String>>();
- ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars );
- for( String val : tmp )
- ret.add(new Tuple2<Long,String>(_workerID, val));
-
- return ret;
- }
-
- /**
- *
- * @param ID
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void configureWorker( long ID )
- throws DMLRuntimeException, DMLUnsupportedOperationException, IOException
- {
- _workerID = ID;
-
- //parse and setup parfor body program
- ParForBody body = ProgramConverter.parseParForBody(_prog, (int)_workerID);
- _childBlocks = body.getChildBlocks();
- _ec = body.getEc();
- _resultVars = body.getResultVarNames();
- _numTasks = 0;
- _numIters = 0;
-
- //init local cache manager
- if( !CacheableData.isCachingActive() ) {
- String uuid = IDHandler.createDistributedUniqueID();
- LocalFileUtils.createWorkingDirectoryWithUUID( uuid );
- CacheableData.initCaching( uuid ); //incl activation, cache dir creation (each map task gets its own dir for simplified cleanup)
- }
- if( !CacheableData.cacheEvictionLocalFilePrefix.contains("_") ){ //account for local mode
- CacheableData.cacheEvictionLocalFilePrefix = CacheableData.cacheEvictionLocalFilePrefix +"_" + _workerID;
- }
-
- //ensure that resultvar files are not removed
- super.pinResultVariables();
-
- //enable/disable caching (if required)
- if( !_caching )
- CacheableData.disableCaching();
-
- //make as lazily intialized
- _initialized = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForUtils.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForUtils.java
deleted file mode 100644
index 78c884b..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParForUtils.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * Common functionalities for parfor workers in MR jobs. Used by worker wrappers in
- * mappers (base RemoteParFor) and reducers (fused data partitioning and parfor)
- *
- */
-public class RemoteParForUtils
-{
-
- /**
- *
- * @param reporter
- * @param deltaTasks
- * @param deltaIterations
- */
- public static void incrementParForMRCounters(Reporter reporter, long deltaTasks, long deltaIterations)
- {
- //report parfor counters
- if( deltaTasks>0 )
- reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMTASKS.toString(), deltaTasks);
- if( deltaIterations>0 )
- reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMITERS.toString(), deltaIterations);
-
- JobConf job = ConfigurationManager.getCachedJobConf();
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- {
- //report cache statistics
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JITCOMPILE.toString(), Statistics.getJITCompileTime());
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_COUNT.toString(), Statistics.getJVMgcCount());
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_TIME.toString(), Statistics.getJVMgcTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_MEM.toString(), CacheStatistics.getMemHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FSBUFF.toString(), CacheStatistics.getFSBuffHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FS.toString(), CacheStatistics.getFSHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_HDFS.toString(), CacheStatistics.getHDFSHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FSBUFF.toString(), CacheStatistics.getFSBuffWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FS.toString(), CacheStatistics.getFSWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_HDFS.toString(), CacheStatistics.getHDFSWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQR.toString(), CacheStatistics.getAcquireRTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQM.toString(), CacheStatistics.getAcquireMTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_RLS.toString(), CacheStatistics.getReleaseTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_EXP.toString(), CacheStatistics.getExportTime());
-
- //reset cache statistics to prevent overlapping reporting
- CacheStatistics.reset();
- }
- }
-
- /**
- *
- * @param workerID
- * @param vars
- * @param resultVars
- * @param out
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public static void exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars, OutputCollector<Writable, Writable> out )
- throws DMLRuntimeException, IOException
- {
- exportResultVariables(workerID, vars, resultVars, null, out);
- }
-
- /**
- * For remote MR parfor workers.
- *
- * @param workerID
- * @param vars
- * @param resultVars
- * @param rvarFnames
- * @param out
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public static void exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars,
- HashMap<String,String> rvarFnames, OutputCollector<Writable, Writable> out )
- throws DMLRuntimeException, IOException
- {
- //create key and value for reuse
- LongWritable okey = new LongWritable( workerID );
- Text ovalue = new Text();
-
- //foreach result variables probe if export necessary
- for( String rvar : resultVars )
- {
- Data dat = vars.get( rvar );
-
- //export output variable to HDFS (see RunMRJobs)
- if ( dat != null && dat.getDataType() == DataType.MATRIX )
- {
- MatrixObject mo = (MatrixObject) dat;
- if( mo.isDirty() )
- {
- if( ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames!=null )
- {
- String fname = rvarFnames.get( rvar );
- if( fname!=null )
- mo.setFileName( fname );
-
- //export result var (iff actually modified in parfor)
- mo.exportData(); //note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
- rvarFnames.put(rvar, mo.getFileName());
- }
- else
- {
- //export result var (iff actually modified in parfor)
- mo.exportData(); //note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
- }
-
- //pass output vars (scalars by value, matrix by ref) to result
- //(only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
- String datStr = ProgramConverter.serializeDataObject(rvar, mo);
- ovalue.set( datStr );
- out.collect( okey, ovalue );
- }
- }
- }
- }
-
- /**
- * For remote Spark parfor workers. This is a simplified version compared to MR.
- *
- * @param workerID
- * @param vars
- * @param resultVars
- * @param rvarFnames
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public static ArrayList<String> exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars)
- throws DMLRuntimeException, IOException
- {
- ArrayList<String> ret = new ArrayList<String>();
-
- //foreach result variables probe if export necessary
- for( String rvar : resultVars )
- {
- Data dat = vars.get( rvar );
-
- //export output variable to HDFS (see RunMRJobs)
- if ( dat != null && dat.getDataType() == DataType.MATRIX )
- {
- MatrixObject mo = (MatrixObject) dat;
- if( mo.isDirty() )
- {
- //export result var (iff actually modified in parfor)
- mo.exportData();
-
-
- //pass output vars (scalars by value, matrix by ref) to result
- //(only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
- ret.add( ProgramConverter.serializeDataObject(rvar, mo) );
- }
- }
- }
-
- return ret;
- }
-
-
- /**
- * Cleanup all temporary files created by this SystemML process
- * instance.
- *
- */
- public static void cleanupWorkingDirectories()
- {
- //use the given job configuration for infrastructure analysis (see configure);
- //this is important for robustness w/ misconfigured classpath which also contains
- //core-default.xml and hence hides the actual cluster configuration; otherwise
- //there is missing cleanup of working directories
- JobConf job = ConfigurationManager.getCachedJobConf();
-
- if( !InfrastructureAnalyzer.isLocalMode(job) )
- {
- //delete cache files
- CacheableData.cleanupCacheDir();
- //disable caching (prevent dynamic eviction)
- CacheableData.disableCaching();
- //cleanup working dir (e.g., of CP_FILE instructions)
- LocalFileUtils.cleanupWorkingDirectory();
- }
- }
-
- /**
- *
- * @param out
- * @return
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public static LocalVariableMap[] getResults( List<Tuple2<Long,String>> out, Log LOG )
- throws DMLRuntimeException
- {
- HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>();
-
- int countAll = 0;
- for( Tuple2<Long,String> entry : out )
- {
- Long key = entry._1();
- String val = entry._2();
- if( !tmp.containsKey( key ) )
- tmp.put(key, new LocalVariableMap ());
- Object[] dat = ProgramConverter.parseDataObject( val );
- tmp.get(key).put((String)dat[0], (Data)dat[1]);
- countAll++;
- }
-
- if( LOG != null ) {
- LOG.debug("Num remote worker results (before deduplication): "+countAll);
- LOG.debug("Num remote worker results: "+tmp.size());
- }
-
- //create return array
- return tmp.values().toArray(new LocalVariableMap[0]);
- }
-}
[25/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropy.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropy.java b/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropy.java
deleted file mode 100644
index df131c9..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropy.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedCrossEntropy extends Lop
-{
- public static final String OPCODE = "mapwcemm";
- public static final String OPCODE_CP = "wcemm";
- private int _numThreads = 1;
-
- public enum WCeMMType {
- BASIC,
- }
-
- private WCeMMType _wcemmType = null;
-
- public WeightedCrossEntropy(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WCeMMType wt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedCeMM, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wcemmType = wt;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedCrossEntropy";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- if( getExecType() == ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wcemmType);
-
- //append degree of parallelism
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( getExecType()==ExecType.MR )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( getExecType()==ExecType.MR )
- return new int[]{2,3};
- else
- return new int[]{-1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropyR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropyR.java b/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropyR.java
deleted file mode 100644
index 58f3cc7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedCrossEntropyR.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedCrossEntropyR extends Lop
-{
-
- public static final String OPCODE = "redwcemm";
-
- private WCeMMType _wcemmType = null;
-
- private boolean _cacheU = false;
- private boolean _cacheV = false;
-
- public WeightedCrossEntropyR(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WCeMMType wt, boolean cacheU, boolean cacheV, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedCeMM, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wcemmType = wt;
- _cacheU = cacheU;
- _cacheV = cacheV;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- */
- private void setupLopProperties( ExecType et )
- throws LopsException
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedCrossEntropyR";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wcemmType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheU);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheV);
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( _cacheU || _cacheV )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( !_cacheU && !_cacheV )
- return new int[]{-1};
- else if( _cacheU && !_cacheV )
- return new int[]{2};
- else if( !_cacheU && _cacheV )
- return new int[]{3};
- else
- return new int[]{2,3};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedDivMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedDivMM.java b/src/main/java/com/ibm/bi/dml/lops/WeightedDivMM.java
deleted file mode 100644
index ab6a858..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedDivMM.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-/**
- *
- */
-public class WeightedDivMM extends Lop
-{
- public static final String OPCODE = "mapwdivmm";
- public static final String OPCODE_CP = "wdivmm";
- private int _numThreads = 1;
-
- public enum WDivMMType {
- DIV_LEFT, //t(t(U) %*% (W / U%*%t(V)))
- DIV_RIGHT, //(W / U%*%t(V)) %*% V
- MULT_BASIC, //(W * U%*%t(V))
- MULT_LEFT, //t(t(U) %*% (W * U%*%t(V)))
- MULT_RIGHT, //(W * U%*%t(V)) %*% V
- MULT_MINUS_LEFT, //t(t(U) %*% ((X!=0) * (U%*%t(V) - X)))
- MULT_MINUS_RIGHT; //((X!=0) * (U%*%t(V) - X)) %*% V
-
- public boolean isBasic(){
- return (this == MULT_BASIC);
- }
- public boolean isLeft() {
- return (this == DIV_LEFT || this == MULT_LEFT || this == MULT_MINUS_LEFT);
- }
- public boolean isRight() {
- return !(isLeft() || isBasic());
- }
- public boolean isMult() {
- return (this == MULT_LEFT || this == MULT_RIGHT || this == MULT_MINUS_LEFT || this == MULT_MINUS_RIGHT);
- }
- public boolean isMinus(){
- return (this == MULT_MINUS_LEFT || this == MULT_MINUS_RIGHT);
- }
- public MatrixCharacteristics computeOutputCharacteristics(long Xrlen, long Xclen, long rank) {
- if( isBasic() )
- return new MatrixCharacteristics( Xrlen, Xclen, -1, -1);
- else
- return new MatrixCharacteristics(isLeft()?Xclen:Xrlen, rank, -1, -1);
- }
- }
-
- private WDivMMType _weightsType = null;
-
- public WeightedDivMM(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WDivMMType wt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedDivMM, dt, vt);
- addInput(input1); //W
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- _weightsType = wt;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedDivMM";
- }
-
- /* MR instruction generation */
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- /* CP/SPARK instruction generation */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- if( getExecType() == ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_weightsType);
-
- //append degree of parallelism
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( getExecType()==ExecType.MR )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( getExecType()==ExecType.MR )
- return new int[]{2,3};
- else
- return new int[]{-1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedDivMMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedDivMMR.java b/src/main/java/com/ibm/bi/dml/lops/WeightedDivMMR.java
deleted file mode 100644
index 8f8522f..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedDivMMR.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedDivMMR extends Lop
-{
- public static final String OPCODE = "redwdivmm";
-
- private WDivMMType _weightsType = null;
- private boolean _cacheU = false;
- private boolean _cacheV = false;
-
- public WeightedDivMMR(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WDivMMType wt, boolean cacheU, boolean cacheV, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedDivMM, dt, vt);
- addInput(input1); //W
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- _weightsType = wt;
- _cacheU = cacheU;
- _cacheV = cacheV;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- */
- private void setupLopProperties( ExecType et )
- throws LopsException
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedDivMMR";
- }
-
- /* MR instruction generation */
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- /* CP/SPARK instruction generation */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_weightsType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheU);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheV);
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( _cacheU || _cacheV )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( !_cacheU && !_cacheV )
- return new int[]{-1};
- else if( _cacheU && !_cacheV )
- return new int[]{2};
- else if( !_cacheU && _cacheV )
- return new int[]{3};
- else
- return new int[]{2,3};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoid.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoid.java b/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoid.java
deleted file mode 100644
index 310ae37..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoid.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedSigmoid extends Lop
-{
-
- public static final String OPCODE = "mapwsigmoid";
- public static final String OPCODE_CP = "wsigmoid";
- private int _numThreads = 1;
-
- public enum WSigmoidType {
- BASIC,
- LOG,
- MINUS,
- LOG_MINUS,
- }
-
- private WSigmoidType _wsigmoidType = null;
-
- public WeightedSigmoid(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WSigmoidType wt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedSigmoid, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wsigmoidType = wt;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedSigmoid";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- if( getExecType() == ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wsigmoidType);
-
- //append degree of parallelism
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( getExecType()==ExecType.MR )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( getExecType()==ExecType.MR )
- return new int[]{2,3};
- else
- return new int[]{-1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoidR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoidR.java b/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoidR.java
deleted file mode 100644
index de99bfd..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedSigmoidR.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedSigmoidR extends Lop
-{
-
- public static final String OPCODE = "redwsigmoid";
-
- private WSigmoidType _wsType = null;
-
- private boolean _cacheU = false;
- private boolean _cacheV = false;
-
- public WeightedSigmoidR(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WSigmoidType wt, boolean cacheU, boolean cacheV, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedSigmoid, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wsType = wt;
- _cacheU = cacheU;
- _cacheV = cacheV;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- */
- private void setupLopProperties( ExecType et )
- throws LopsException
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedSigmoidR";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wsType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheU);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheV);
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( _cacheU || _cacheV )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( !_cacheU && !_cacheV )
- return new int[]{-1};
- else if( _cacheU && !_cacheV )
- return new int[]{2};
- else if( !_cacheU && _cacheV )
- return new int[]{3};
- else
- return new int[]{2,3};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLoss.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLoss.java b/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLoss.java
deleted file mode 100644
index 59b7630..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLoss.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedSquaredLoss extends Lop
-{
- public static final String OPCODE = "mapwsloss";
- public static final String OPCODE_CP = "wsloss";
- private int _numThreads = 1;
-
- public enum WeightsType {
- POST,
- POST_NZ,
- PRE,
- NONE;
-
- public boolean hasFourInputs() {
- return (this == POST || this == PRE);
- }
- }
-
- private WeightsType _weightsType = null;
-
- public WeightedSquaredLoss(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, WeightsType wt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedSquaredLoss, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- addInput(input4); //W
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
- input4.addOutput(this);
-
- //setup mapmult parameters
- _weightsType = wt;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedSquaredLoss";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int input4, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(input4),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String input4, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- if( getExecType() == ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(3).prepInputOperand(input4));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_weightsType);
-
- //append degree of parallelism
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( getExecType()==ExecType.MR )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( getExecType()==ExecType.MR )
- return new int[]{2,3};
- else
- return new int[]{-1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLossR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLossR.java b/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLossR.java
deleted file mode 100644
index dcf87a7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedSquaredLossR.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedSquaredLossR extends Lop
-{
- public static final String OPCODE = "redwsloss";
-
- private WeightsType _weightsType = null;
- private boolean _cacheU = false;
- private boolean _cacheV = false;
-
- public WeightedSquaredLossR(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, WeightsType wt, boolean cacheU, boolean cacheV, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedSquaredLoss, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- addInput(input4); //W
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
- input4.addOutput(this);
-
- //setup mapmult parameters
- _weightsType = wt;
- _cacheU = cacheU;
- _cacheV = cacheV;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- */
- private void setupLopProperties( ExecType et )
- throws LopsException
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedSquaredLossR";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int input4, int output)
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(input4),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String input4, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(3).prepInputOperand(input4));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_weightsType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheU);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheV);
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( _cacheU || _cacheV )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( !_cacheU && !_cacheV )
- return new int[]{-1};
- else if( _cacheU && !_cacheV )
- return new int[]{2};
- else if( !_cacheU && _cacheV )
- return new int[]{3};
- else
- return new int[]{2,3};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMM.java b/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMM.java
deleted file mode 100644
index 48269df..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMM.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.Unary.OperationTypes;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedUnaryMM extends Lop
-{
- public static final String OPCODE = "mapwumm";
- public static final String OPCODE_CP = "wumm";
-
- public enum WUMMType {
- MULT,
- DIV,
- }
-
- private WUMMType _wummType = null;
- private OperationTypes _uop = null;
- private int _numThreads = 1;
-
- public WeightedUnaryMM(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WUMMType wt, OperationTypes op, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedUMM, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wummType = wt;
- _uop = op;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedUMM";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- if( getExecType() == ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(Unary.getOpcode(_uop));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wummType);
-
- //append degree of parallelism
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( getExecType()==ExecType.MR )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( getExecType()==ExecType.MR )
- return new int[]{2,3};
- else
- return new int[]{-1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMMR.java b/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMMR.java
deleted file mode 100644
index 362cb4d..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/WeightedUnaryMMR.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.Unary.OperationTypes;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class WeightedUnaryMMR extends Lop
-{
- public static final String OPCODE = "redwumm";
-
- private WUMMType _wummType = null;
- private OperationTypes _uop = null;
- private boolean _cacheU = false;
- private boolean _cacheV = false;
-
- public WeightedUnaryMMR(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, WUMMType wt, OperationTypes op, boolean cacheU, boolean cacheV, ExecType et)
- throws LopsException
- {
- super(Lop.Type.WeightedUMM, dt, vt);
- addInput(input1); //X
- addInput(input2); //U
- addInput(input3); //V
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _wummType = wt;
- _uop = op;
- _cacheU = cacheU;
- _cacheV = cacheV;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- */
- private void setupLopProperties( ExecType et )
- throws LopsException
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "Operation = WeightedUMMR";
- }
-
- @Override
- public String getInstructions(int input1, int input2, int input3, int output)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input1),
- String.valueOf(input2),
- String.valueOf(input3),
- String.valueOf(output));
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(Unary.getOpcode(_uop));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_wummType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheU);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheV);
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- if( _cacheU || _cacheV )
- return true;
- else
- return false;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( !_cacheU && !_cacheV )
- return new int[]{-1};
- else if( _cacheU && !_cacheV )
- return new int[]{2};
- else if( !_cacheU && _cacheV )
- return new int[]{3};
- else
- return new int[]{2,3};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/ZeroOut.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/ZeroOut.java b/src/main/java/com/ibm/bi/dml/lops/ZeroOut.java
deleted file mode 100644
index 6494c3e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/ZeroOut.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-public class ZeroOut extends Lop
-{
-
-
- private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, long rowDim, long colDim, DataType dt, ValueType vt, ExecType et) {
- this.addInput(inputMatrix);
- this.addInput(rowL);
- this.addInput(rowU);
- this.addInput(colL);
- this.addInput(colU);
-
- inputMatrix.addOutput(this);
- rowL.addOutput(this);
- rowU.addOutput(this);
- colL.addOutput(this);
- colU.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- this.lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public ZeroOut(
- Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, long rowDim, long colDim, DataType dt, ValueType vt)
- throws LopsException {
- super(Lop.Type.ZeroOut, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, ExecType.MR);
- }
-
- public ZeroOut(
- Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, long rowDim, long colDim, DataType dt, ValueType vt, ExecType et)
- throws LopsException {
- super(Lop.Type.ZeroOut, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et);
- }
-
- private String getOpcode() {
-
- return "zeroOut";
- }
-
- @Override
- public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- // coll, colu
- sb.append( getInputs().get(3).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(4).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index)
- throws LopsException {
- /*
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
- */
-
- return getInstructions(Integer.toString(input_index1), input_index2+"", input_index3+"", input_index4+"", input_index5+"", Integer.toString(output_index));
- }
-
- @Override
- public String toString() {
- return "ZeroOut";
- }
-}
[46/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerInterface.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerInterface.java
deleted file mode 100644
index b2bab75..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerInterface.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-//import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-
-
-/**
- * Class implements a command line interface (CLI) for the DML language debugger
- */
-public class DMLDebuggerInterface
-{
-
- //SystemML debugger functionality options
- private Options options;
-
- /**
- * Constructor for a DML debugger CLI
- */
- public DMLDebuggerInterface() {
- options = new Options();
- }
-
- /**
- * Returns the debugger CLI options
- * @return options CLI options
- */
- public Options getOptions()
- {
- return options;
- }
-
- /**
- * Set DML debugger CLI functionality menu
- */
- @SuppressWarnings("static-access")
- public void setOptions()
- {
- //add help option
- options.addOption("h", "help", false, "list debugger functions");
-
- //add run option
- options.addOption("r", "run", false, "start your DML script");
-
- //add quit option
- options.addOption("q", "quit", false, "exit debug mode");
-
- //add resume option
- options.addOption("c", "continue", false, "continue running your DML script");
-
- //add step over
- //options.addOption("n", "next", false, "next line, stepping over function calls");
-
- //add single-stepping
- options.addOption("s", "step", false, "next line, stepping into function calls");
-
- //add single-stepping
- options.addOption("si", "stepi", false, "next runtime instruction rather than DML source lines (for advanced users)");
-
- // No step return for now
- //add step return
-// Option stepReturn = OptionBuilder.withArgName( "function-name" )
-// .hasOptionalArg()
-// .withDescription( "execute instructions associated with current function as single step")
-// .create( "step_return" );
-// options.addOption(stepReturn);
-
- //add set breakpoint option
- Option setBreakpoint = OptionBuilder.withLongOpt( "break" )
- .withArgName( "line-number" )
- .hasArg()
- .withDescription( "set breakpoint at given line number" )
- .create( "b" );
- options.addOption(setBreakpoint);
-
- // The key assumption here is that user doesnot keep toggling breakpoints too often
- //add delete breakpoint option
- Option disableBreakpoint = OptionBuilder.withLongOpt( "delete" )
- .withArgName( "line-number" )
- .hasArg()
- .withDescription( "delete breakpoint at given line number" )
- .create( "d" );
- options.addOption(disableBreakpoint);
-
- //add list breakpoints option
- Option infoOption = OptionBuilder.withLongOpt( "info" )
- .withArgName( "[break | frame]" )
- .hasOptionalArgs(1)
- .withDescription( "show all breakpoints or frames (info <break | frame>)" )
- .create( "i" );
- options.addOption(infoOption);
-
- //add display DML script option
- Option displayScript = OptionBuilder.withLongOpt( "list" )
- .withArgName( "[next numlines] | [prev numlines] | [all]" )
- .hasOptionalArgs(2)
- .withValueSeparator(' ')
- .withDescription( "display DML script source lines. Default: numlines = 10" )
- .create( "l" );
- options.addOption(displayScript);
-
- //add display DML script interspersed with runtime instructions option
- Option displayInst = OptionBuilder.withLongOpt( "listi" )
- .withArgName( "[next numlines] | [prev numlines] | [all]" )
- .hasOptionalArgs(2)
- .withValueSeparator(' ')
- .withDescription( "display corresponding instructions for DML script source lines. Default: numlines = 10 (for advanced users)" )
- .create( "li" );
- options.addOption(displayInst);
-
- //add set value of DML scalar variable option
- Option setVar = OptionBuilder.withArgName( "varName value" )
- .hasArgs(2)
- .withValueSeparator(' ')
- .withDescription( "set value of a scalar or specified cell of a matrix variable. (Eg: \'set alpha 0.1\' or \'set A[1,2] 20\')" )
- .create( "set" );
- options.addOption(setVar);
-
- //add display DML matrix (or vector) variable option
- Option displayMatrix = OptionBuilder.withLongOpt( "print" )
- .withArgName( "varName" )
- .hasArg()
- .withDescription( "display contents of a scalar or matrix variable or rows/columns/cell of matrix. (Eg: \'p alpha\' or \'p A\' or \'p A[1,]\')" )
- .create( "p" );
- options.addOption(displayMatrix);
-
- Option displayTypeMatrix = OptionBuilder //.withLongOpt( "whatis" )
- .withArgName( "varName" )
- .hasArg()
- .withDescription( "display the type (and metadata) of a variable. (Eg: \'whatis alpha\' or \'whatis A\')" )
- .create( "whatis" );
- options.addOption(displayTypeMatrix);
- }
-
- /**
- * Add new function to debugger CLI functions menu
- * @param opt New debugging function option
- */
- public void setOption(Option opt)
- {
- this.options.addOption(opt);
- }
-
- public void writeToStandardOutput(String outputStr) {
- System.out.print(outputStr);
- System.out.flush();
-// if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
-// System.out.print(outputStr);
-// System.out.flush();
-// }
-// else {
-// // TODO: Write to client socket that was created by server socket
-// }
- }
-
- public void writelnToStandardOutput(String outputStr) {
- System.out.println(outputStr);
- System.out.flush();
-// if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
-// System.out.println(outputStr);
-// System.out.flush();
-// }
-// else {
-// // TODO: Write to client socket that was created by server socket
-// }
- }
-
- public void writeToStandardError(String errStr) {
- System.err.print(errStr);
- System.err.flush();
-// if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
-// System.err.print(errStr);
-// System.err.flush();
-// }
-// else {
-// // TODO: Write to client socket that was created by server socket
-// }
- }
-
-
- public void writelnToStandardError(String errStr) {
- System.err.println(errStr);
- System.err.flush();
-// if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
-// System.err.println(errStr);
-// System.err.flush();
-// }
-// else {
-// // TODO: Write to client socket that was created by server socket
-// }
- }
-
-
- /**
- * Display debugger usage/help info
- */
- public void getDebuggerCLI()
- {
- // if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
- // Not using formatter because it outputs -h, -s commands. But, I still want to use GnuParser
- // HelpFormatter debuggerUsage = new HelpFormatter();
- // debuggerUsage.setLongOptPrefix("-"); //Eliminates the use of "--" for alternate commands
- // debuggerUsage.setWidth(125); //Enables readability of commands description
- // debuggerUsage.printHelp( "SystemMLdb <command> ", "\nSystemMLdb commands:\n", options,
- // "\n\nSystemMLdb is a prototype debugger for SystemML. There is NO warranty as "
- // + "it is still in experimental state.\n\n" );
- String helpString = "SystemMLdb commands:"
- // "usage: SystemMLdb <command>\n\nSystemMLdb commands:\n"
- + "\nh,help list debugger functions"
- + "\nr,run start your DML script"
- + "\nq,quit exit debug mode"
- + "\nc,continue continue running your DML script"
- //
- + "\nl,list <[next numlines] | [prev numlines] | [all]> display DML script source lines. Default: numlines = 10"
- + "\nb,break <line-number> set breakpoint at given line number"
- + "\nd,delete <line-number> delete breakpoint at given line number"
- + "\ns,step next line, stepping into function calls"
- + "\ni,info <break | frame> show all breakpoints or frames (info <break | frame>)"
- //
- + "\np,print <varName> display contents of a scalar or matrix variable or"
- + "\n rows/columns/cell of matrix. (Eg: \'p alpha\' or \'p A\' or \'p A[1,]\')"
- + "\nset <varName value> set value of a scalar or specified cell of a matrix variable. (Eg:"
- + "\n \'set alpha 0.1\' or \'set A[1,2] 20\')"
- + "\nwhatis <varName> display the type (and metadata) of a variable. (Eg: \'whatis alpha\'"
- + "\n or \'whatis A\')"
- + "\nli,listi <[next numlines] | [prev numlines] | [all]> display corresponding instructions for DML script source lines."
- + "\n Default: numlines = 10 (for advanced users)"
- + "\nsi,stepi next runtime instruction rather than DML source lines (for advanced"
- + "\n users)"
- //
- + "\n"
- ;
- writelnToStandardOutput(helpString);
-
-// }
-// else {
-// // TODO: Write to client socket that was created by server socket
-// // printHelp(PrintWriter pw, int width, String cmdLineSyntax, String header, Options options, int leftPad, int descPad, String footer)
-// }
-
- }
-
- /**
- * Read, process and return command from debugger CLI
- * @return CommandLine Current debug command (enter by user)
- * @throws DMLDebuggerException
- */
- public CommandLine getDebuggerCommand()
- throws DMLDebuggerException
- {
- CommandLine cmd = null;
- String [] args = null;
-
-// if(!DMLScript.ENABLE_SERVER_SIDE_DEBUG_MODE) {
- //Display input prompt
- writeToStandardOutput("(SystemMLdb) ");
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
- try {
- //read command line argument(s)
- // To add 'up/down'-feature, use jline library
- String line = br.readLine();
- if(line != null && !line.isEmpty() ) {
- args = line.split(" ");
- if(args[0].startsWith("-")) {
- // So as to avoid parsing '-i' command
- writelnToStandardError("Error reading command line arguments. Try \"help\".");
- return cmd;
- }
- args[0] = "-" + args[0];
- }
- } catch (IOException ae) {
- writelnToStandardError("Error reading command line arguments. Try \"help\".");
- return cmd;
- }
-//
-// }
-// else {
-// // TODO: Read commands from Process that was created by server socket
-// }
-
- CommandLineParser CLIparser = new GnuParser();
- try {
- //parse current command
- cmd = CLIparser.parse(getOptions(), args);
- } catch (ParseException pe) {
- System.err.println("Undefined command (or command arguments). Try \"help\".");
- }
- return cmd;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerProgramInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerProgramInfo.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerProgramInfo.java
deleted file mode 100644
index f90ca49..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerProgramInfo.java
+++ /dev/null
@@ -1,375 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-
-/**
- * This class contains the parsed and compiled DML script w/ hops, lops and runtime program.
- * Additionally, it provides an interface for managing breakpoints and disassembling a DML script.
- * Note: ONLY USED FOR DEBUGGING PURPOSES
- */
-public class DMLDebuggerProgramInfo
-{
-
- public DMLConfig conf; //DML configuration information
- public DMLProgram prog; //DML program representation
- public DMLTranslator dmlt; //DML program hops and lops, rewrites
- public Program rtprog; //DML runtime program
-
- private TreeMap<Integer, ArrayList<Instruction>> disassembler; //map between DML program line numbers and corresponding runtime instruction(s)
-
- private int prevLineNum = 0; //used for approximating line numbers for instructions (if necessary)
- private long instID = 1; //runtime instruction ID
-
- private String location=null; //DML program namespace and function name
-
- /**
- * Constructor for DMLDebuggerProgramInfo class.
- */
- public DMLDebuggerProgramInfo() {
- disassembler = new TreeMap<Integer, ArrayList<Instruction>>();
- }
-
- /**
- * Getter for DML script to instruction map field
- * @return DMLInstMap Map between DML script line numbers and runtime instruction(s)
- */
- public TreeMap<Integer, ArrayList<Instruction>> getDMLInstMap() {
- this.setDMLInstMap();
- return disassembler;
- }
-
- /**
- * Access breakpoint instruction at specified line number in runtime program (if valid)
- * @param lineNumber Location for breakpoint operation
- * @param op Breakpoint operation (op=0 for create, op=1 for enable/disable, op=2 for delete)
- * @param status Current breakpoint status
- */
- public void accessBreakpoint(int lineNumber, int op, BPINSTRUCTION_STATUS status)
- {
- if (this.rtprog != null)
- {
- //Functions: For each function program block (if any), get instructions corresponding to each line number
- HashMap<String,FunctionProgramBlock> funcMap = this.rtprog.getFunctionProgramBlocks();
- if (funcMap != null && !funcMap.isEmpty() ) {
- for (Entry<String, FunctionProgramBlock> e : funcMap.entrySet())
- {
- location = e.getKey();
- FunctionProgramBlock fpb = e.getValue();
- if(fpb instanceof ExternalFunctionProgramBlock)
- continue;
- else
- {
- for (ProgramBlock pb : fpb.getChildBlocks())
- accessProgramBlockBreakpoint(pb, lineNumber, op, status);
- }
- }
- }
- //Main program: for each program block, get instructions corresponding to current line number (if any)
- location=DMLProgram.constructFunctionKey(DMLProgram.DEFAULT_NAMESPACE, "main");
- for (ProgramBlock pb : this.rtprog.getProgramBlocks())
- {
- if (pb != null)
- accessProgramBlockBreakpoint(pb, lineNumber, op, status);
- }
- }
- }
-
- /**
- * Access breakpoint instruction at specified line number in program block (if valid)
- * @param pb Current program block
- * @param lineNumber Location for inserting breakpoint
- * @param status Current breakpoint status
- */
- private void accessProgramBlockBreakpoint(ProgramBlock pb, int lineNumber, int op, BPINSTRUCTION_STATUS status)
- {
- if (pb instanceof FunctionProgramBlock)
- {
- FunctionProgramBlock fpb = (FunctionProgramBlock)pb;
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- accessProgramBlockBreakpoint(pbc, lineNumber, op, status);
- }
- else if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- this.accesBreakpointInstruction(wpb.getPredicate(), lineNumber, op, status);
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- accessProgramBlockBreakpoint(pbc, lineNumber, op, status);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- this.accesBreakpointInstruction(ipb.getPredicate(), lineNumber, op, status);
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- accessProgramBlockBreakpoint(pbc, lineNumber, op, status);
- if( !ipb.getChildBlocksElseBody().isEmpty() )
- {
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- accessProgramBlockBreakpoint(pbc, lineNumber, op, status);
- }
- }
- else if (pb instanceof ForProgramBlock) //incl parfor
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- this.accesBreakpointInstruction(fpb.getFromInstructions(), lineNumber, op, status);
- this.accesBreakpointInstruction(fpb.getToInstructions(), lineNumber, op, status);
- this.accesBreakpointInstruction(fpb.getIncrementInstructions(), lineNumber, op, status);
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- accessProgramBlockBreakpoint(pbc, lineNumber, op, status);
- }
- else
- {
- this.accesBreakpointInstruction(pb.getInstructions(), lineNumber, op, status);
- }
- }
-
- /**
- * Access breakpoint instruction at specified line number in set of instructions (if valid)
- * @param instructions Instructions for current program block
- * @param lineNumber Location for inserting breakpoint
- * @param status Current breakpoint status
- */
- private void accesBreakpointInstruction(ArrayList<Instruction> instructions, int lineNumber, int op, BPINSTRUCTION_STATUS status)
- {
- for (int i = 0; i < instructions.size() ; i++)
- {
- Instruction currInst = instructions.get(i);
- if (op == 0)
- {
- if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- // Check if current instruction line number correspond to breakpoint line number
- if (currMRInst.findMRInstructions(lineNumber)) {
- BreakPointInstruction breakpoint = new BreakPointInstruction();
- breakpoint.setLocation(currInst);
- breakpoint.setInstID(instID++);
- breakpoint.setBPInstructionLocation(location);
- instructions.add(i, breakpoint);
- DMLBreakpointManager.insertBreakpoint(breakpoint, lineNumber);
- return;
- }
- }
- else if (currInst instanceof CPInstruction || currInst instanceof SPInstruction)
- {
- // Check if current instruction line number correspond to breakpoint line number
- if (currInst.getLineNum() == lineNumber) {
- BreakPointInstruction breakpoint = new BreakPointInstruction();
- breakpoint.setLocation(currInst);
- breakpoint.setInstID(instID++);
- breakpoint.setBPInstructionLocation(location);
- instructions.add(i, breakpoint);
- DMLBreakpointManager.insertBreakpoint(breakpoint, lineNumber);
- return;
- }
- }
- else if (currInst instanceof BreakPointInstruction && currInst.getLineNum() == lineNumber)
- {
- BreakPointInstruction breakpoint = (BreakPointInstruction) currInst;
- breakpoint.setBPInstructionStatus(BPINSTRUCTION_STATUS.ENABLED);
- breakpoint.setBPInstructionLocation(location);
- instructions.set(i, breakpoint);
- DMLBreakpointManager.updateBreakpoint(lineNumber, status);
- return;
- }
- }
- else
- {
- // Check if current instruction line number correspond to breakpoint line number
- if (currInst instanceof BreakPointInstruction && currInst.getLineNum() == lineNumber)
- {
- if (op == 1)
- {
- BreakPointInstruction breakpoint = (BreakPointInstruction) currInst;
- breakpoint.setLocation(currInst);
- breakpoint.setInstID(currInst.getInstID());
- breakpoint.setBPInstructionStatus(status);
- breakpoint.setBPInstructionLocation(location);
- instructions.set(i, breakpoint);
- DMLBreakpointManager.updateBreakpoint(lineNumber, status);
- }
- else
- {
- instructions.remove(i);
- DMLBreakpointManager.removeBreakpoint(lineNumber, status);
- }
- return;
- }
- }
- }
- }
-
- /**
- * Getter for DML instruction map field (i.e. disassembler functionality)
- * For each DML script line, get runtime instructions (if any)
- */
- public void setDMLInstMap()
- {
- disassembler = new TreeMap<Integer, ArrayList<Instruction>>();
- if (this.rtprog != null)
- {
- //Functions: For each function program block (if any), get instructions corresponding to each line number
- HashMap<String,FunctionProgramBlock> funcMap = this.rtprog.getFunctionProgramBlocks();
- if (funcMap != null && !funcMap.isEmpty() )
- {
- for (Entry<String, FunctionProgramBlock> e : funcMap.entrySet())
- {
- FunctionProgramBlock fpb = e.getValue();
- if(fpb instanceof ExternalFunctionProgramBlock)
- continue;
- else
- {
- for (ProgramBlock pb : fpb.getChildBlocks())
- setProgramBlockInstMap(pb);
- }
- }
- }
- //Main program: for each program block, get instructions corresponding to current line number (if any)
- for (ProgramBlock pb : this.rtprog.getProgramBlocks())
- {
- if (pb != null)
- setProgramBlockInstMap(pb);
- }
- }
- }
-
- /**
- * For each program block, get runtime instructions (if any)
- * @param pb Current program block
- */
- private void setProgramBlockInstMap(ProgramBlock pb)
- {
- if (pb instanceof FunctionProgramBlock)
- {
- FunctionProgramBlock fpb = (FunctionProgramBlock)pb;
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- this.setInstMap(wpb.getPredicate());
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- this.setInstMap(ipb.getPredicate());
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- setProgramBlockInstMap(pbc);
- if( !ipb.getChildBlocksElseBody().isEmpty() )
- {
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- setProgramBlockInstMap(pbc);
- }
- }
- else if (pb instanceof ForProgramBlock) //incl parfor
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- this.setInstMap(fpb.getFromInstructions());
- this.setInstMap(fpb.getToInstructions());
- this.setInstMap(fpb.getIncrementInstructions());
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else
- {
- this.setInstMap(pb.getInstructions());
- }
- }
-
- /**
- * For each instruction, generate map with corresponding DML
- * script line number
- * @param instructions Instructions for current program block
- */
- private void setInstMap(ArrayList<Instruction> instructions)
- {
- for (int i = 0; i < instructions.size() ; i++)
- {
- Instruction currInst = instructions.get(i);
- //set instruction unique identifier
- if (currInst.getInstID() == 0) {
- currInst.setInstID(instID++);
- }
- if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- int min = Integer.MAX_VALUE;
- //iterate of MR job instructions to identify minimum line number
- for (Integer lineNumber : currMRInst.getMRJobInstructionsLineNumbers())
- {
- if (lineNumber < min)
- min = lineNumber;
- }
- //set MR job line number
- if (min == 0 || min == Integer.MAX_VALUE)
- currMRInst.setLocation(prevLineNum, prevLineNum, -1, -1); //last seen instruction line number
- else
- currMRInst.setLocation(min, min, -1, -1); //minimum instruction line number for this MR job
- //insert current MR instruction into corresponding source code line
- if (!disassembler.containsKey(currMRInst.getLineNum()))
- disassembler.put(currMRInst.getLineNum(), new ArrayList<Instruction>());
- disassembler.get(currMRInst.getLineNum()).add(currMRInst);
- }
- else if (currInst instanceof CPInstruction || currInst instanceof SPInstruction)
- {
- //if CP instruction line number is not set, then approximate to last seen line number
- if (currInst.getLineNum() == 0)
- currInst.setLocation(prevLineNum, prevLineNum, -1, -1);
- //insert current CP instruction into corresponding source code line
- if (!disassembler.containsKey(currInst.getLineNum()))
- disassembler.put(currInst.getLineNum(), new ArrayList<Instruction>());
- disassembler.get(currInst.getLineNum()).add(currInst);
- }
- else if (currInst instanceof BreakPointInstruction)
- {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- //insert current BP instruction into corresponding source code line
- if (!disassembler.containsKey(currBPInst.getLineNum()))
- disassembler.put(currBPInst.getLineNum(), new ArrayList<Instruction>());
- disassembler.get(currInst.getLineNum()).add(currBPInst);
- }
- //save instruction's line number as last seen
- if (currInst.getLineNum() != 0)
- prevLineNum = currInst.getLineNum();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDisassemble.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDisassemble.java b/src/main/java/com/ibm/bi/dml/debug/DMLDisassemble.java
deleted file mode 100644
index 1e1838a..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDisassemble.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-
-
-/**
- * Class to produce mapping between source file lines and runtime instructions
- */
-public class DMLDisassemble
-{
- Program _prog;
- //Map between DML program line numbers and corresponding runtime instruction(s)
- TreeMap<Integer, ArrayList<Instruction>> DMLInstMap;
-
- /**
- * Constructor for DML disassembler class
- * @param prog Program reference (e.g., function repository)
- */
- public DMLDisassemble(Program prog)
- {
- _prog = prog;
- DMLInstMap = new TreeMap<Integer, ArrayList<Instruction>>();
- }
-
- ////////////////////////////////////////////////
- // getter and setter functionality
- ////////////////////////////////////////////////
-
- /**
- * Getter for program reference field
- * @return _prog Program reference
- */
- public Program getProgram()
- {
- return _prog;
- }
-
- /**
- * Getter for DML script to instruction map field
- * @return DMLInstMap Map between DML script line numbers and runtime instruction(s)
- */
- public TreeMap<Integer, ArrayList<Instruction>> getDMLInstMap()
- {
- return DMLInstMap;
- }
-
- /**
- * Setter for program reference field
- * @param prog Program reference
- */
- public void setProgram(Program prog)
- {
- this._prog = prog;
- }
-
- /**
- * Setter for DML instruction map field (i.e. disassembler functionality)
- * For each DML script line, get runtime instructions (if any)
- */
- public void setDMLInstMap()
- {
- DMLInstMap = new TreeMap<Integer, ArrayList<Instruction>>();
- if (_prog != null)
- {
- //Functions: For each function program block (if any), get instructions corresponding to each line number
- HashMap<String,FunctionProgramBlock> funcMap = this._prog.getFunctionProgramBlocks();
- if (funcMap != null && !funcMap.isEmpty() )
- {
- for (Entry<String, FunctionProgramBlock> e : funcMap.entrySet())
- {
- String fkey = e.getKey();
- FunctionProgramBlock fpb = e.getValue();
- if(fpb instanceof ExternalFunctionProgramBlock)
- System.err.println("----EXTERNAL FUNCTION "+fkey+"\n");
- else
- {
- System.err.println("----FUNCTION "+fkey+"\n");
- for (ProgramBlock pb : fpb.getChildBlocks())
- setProgramBlockInstMap(pb);
- }
- }
- }
- //Main program: for each program block, get instructions corresponding to current line number (if any)
- for (ProgramBlock pb : this._prog.getProgramBlocks())
- {
- if (pb != null)
- setProgramBlockInstMap(pb);
- }
- }
- }
-
- /**
- * For each program block, get runtime instructions (if any)
- * @param pb Current program block
- */
- private void setProgramBlockInstMap(ProgramBlock pb) {
- if (pb instanceof FunctionProgramBlock)
- {
- FunctionProgramBlock fpb = (FunctionProgramBlock)pb;
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- this.setInstMap(wpb.getPredicate());
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- this.setInstMap(ipb.getPredicate());
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- setProgramBlockInstMap(pbc);
- if( !ipb.getChildBlocksElseBody().isEmpty() )
- {
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- setProgramBlockInstMap(pbc);
- }
- }
- else if (pb instanceof ForProgramBlock) //incl parfor
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- this.setInstMap(fpb.getFromInstructions());
- this.setInstMap(fpb.getToInstructions());
- this.setInstMap(fpb.getIncrementInstructions());
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else
- {
- this.setInstMap(pb.getInstructions());
- }
- }
-
- /**
- * For each instruction, generate map with corresponding DML
- * script line number
- * @param instructions Instructions for current program block
- */
- private void setInstMap(ArrayList<Instruction> instructions ) {
- for (int i = 0; i < instructions.size() ; i++) {
- Instruction currInst = instructions.get(i);
- if (currInst instanceof MRJobInstruction) {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- for (Integer lineNumber : currMRInst.getMRJobInstructionsLineNumbers()) {
- if (!DMLInstMap.containsKey(lineNumber))
- DMLInstMap.put(lineNumber, new ArrayList<Instruction>());
- DMLInstMap.get(lineNumber).add(currMRInst);
- }
- }
- else if (currInst instanceof CPInstruction) {
- CPInstruction currCPInst = (CPInstruction) currInst;
- // Check if current instruction line number correspond to source line number
- if (!DMLInstMap.containsKey(currInst.getLineNum()))
- DMLInstMap.put(currInst.getLineNum(), new ArrayList<Instruction>());
- DMLInstMap.get(currInst.getLineNum()).add(currCPInst);
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- // Check if current instruction line number correspond to source line number
- if (!DMLInstMap.containsKey(currInst.getLineNum()))
- DMLInstMap.put(currInst.getLineNum(), new ArrayList<Instruction>());
- DMLInstMap.get(currInst.getLineNum()).add(currBPInst);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLFrame.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLFrame.java b/src/main/java/com/ibm/bi/dml/debug/DMLFrame.java
deleted file mode 100644
index a43f269..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLFrame.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-
-public class DMLFrame {
-
- private LocalVariableMap variables; //Symbol table of frame variables
- private DMLProgramCounter pc; //Program counter of frame upon return
-
- /**
- * Constructor for a DML frame
- * @param vars Current frame variables (local symbol table)
- * @param pc Current frame program counter
- */
- public DMLFrame(LocalVariableMap vars, DMLProgramCounter pc) {
- variables = vars;
- this.pc = pc;
- }
-
- /**
- * Getter for variables field
- * @return Frame's symbol table
- */
- public LocalVariableMap getVariables() {
- return variables;
- }
-
- /**
- * Getter program counter field
- * @return Frame's program counter
- */
- public DMLProgramCounter getPC() {
- return pc;
- }
-
- /**
- * Setter for variables field
- * @param vars Frame's local variables
- */
- public void setVariables(LocalVariableMap vars) {
- this.variables = vars;
- }
-
- /**
- * Setter for program counter field
- * @param currPC Frame's program counter
- */
- public void setPC(DMLProgramCounter currPC) {
- this.pc = currPC;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLProgramCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLProgramCounter.java b/src/main/java/com/ibm/bi/dml/debug/DMLProgramCounter.java
deleted file mode 100644
index 8f5dc80..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLProgramCounter.java
+++ /dev/null
@@ -1,208 +0,0 @@
-
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import com.ibm.bi.dml.parser.DMLProgram;
-
-
-public class DMLProgramCounter {
-
- private String namespace; //currently executing namespace
- private String fname; //currently executing function name
- private int programBlockNumber; //currently executing program block number within current function
- private int instNumber; //currently executing instruction number within current program block
-
- private long instID; //currently executing instruction
- private int lineNumber; //currently executing line number (not always correctly set)
-
- /**
- * Constructor for DML pc
- */
- public DMLProgramCounter() {
- instID = 0;
- lineNumber = 0;
- }
-
- /**
- * Basic parameterized constructor for DML pc
- * @param name Current namespace
- * @param fn Current function name in namespace
- * @param blockNum Current program block within function
- * @param instNum Current instruction within program block
- */
- public DMLProgramCounter(String name, String fn, int blockNum, int instNum) {
- this();
- namespace = name;
- fname = fn;
- programBlockNumber = blockNum;
- instNumber = instNum;
- }
-
- /**
- * Parameterized constructor for DML pc without line number information
- * @param name Current namespace
- * @param fn Current function name in namespace
- * @param blockNum Current program block within function
- * @param instNum Current instruction within program block
- * @param instID Current instruction unique identifier
- */
- public DMLProgramCounter(String name, String fn, int blockNum, int instNum, long instID) {
- this(name, fn, blockNum, instNum);
- this.instID = instID;
- }
-
- /**
- * Fully parameterized constructor for DML pc
- * @param name Current namespace
- * @param fn Current function name in namespace
- * @param blockNum Current program block within function
- * @param instNum Current instruction within program block
- * @param instID Current instruction unique identifier
- * @param lineNum Current line number
- */
- public DMLProgramCounter(String name, String fn, int blockNum, int instNum, long instID, int lineNum) {
- this(name, fn, blockNum, instNum, instID);
- lineNumber = lineNum;
- }
-
- /**
- * Getter for namespace field
- * @return Current pc's namespace
- */
- public String getNamespace() {
- return namespace;
- }
-
- /**
- * Getter for function name field
- * @return Current pc's function name
- */
- public String getFunctionName() {
- return fname;
- }
-
- /**
- * Getter for program block number field
- * @return Current pc's program block number
- */
- public int getProgramBlockNumber() {
- return programBlockNumber;
- }
-
- /**
- * Getter for instruction number field
- * @return Current pc's instruction number
- */
- public int getInstNumber() {
- return instNumber;
- }
-
- /**
- * Getter for instruction unique identifier field
- * @return Current pc's instruction ID
- */
- public long getInstID() {
- return instID;
- }
-
- /**
- * Getter for line number field
- * @return Current pc's line number
- */
- public int getLineNumber() {
- return lineNumber;
- }
-
- /**
- * Setter for namespace field
- * @param name Current pc's namespace
- */
- public void setNamespace(String name) {
- namespace = name;
- }
- /**
- * Setter for function name field
- * @param fn Current pc's function name
- */
- public void setFunctionName(String fn) {
- fname = fn;
- }
-
- /**
- * Setter for program block number field
- * @param blockNum Current pc's program block number
- */
- public void setProgramBlockNumber(int blockNum) {
- programBlockNumber = blockNum;
- }
-
- /**
- * Setter for instruction number field
- * @param instNum Current pc's instruction number
- */
- public void setInstNumber(int instNum) {
- instNumber = instNum;
- }
-
- /**
- * Setter for instruction unique identifier field
- * @param instID Current pc's instruction unique ID
- */
- public void setInstID(long instID) {
- this.instID = instID;
- }
-
- /**
- * Setter for line number field
- * @param lineNum Current pc's line number
- */
- public void setLineNumber(int lineNum) {
- lineNumber = lineNum;
- }
-
- /**
- * Displays a pretty-printed program counter
- * @return Current pc
- */
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(DMLProgram.constructFunctionKey(this.namespace, this.fname));
- sb.append(" instID ");
- sb.append(this.instID);
- sb.append(": (line ");
- sb.append(this.lineNumber);
- sb.append(")");
- return sb.toString();
- }
-
- /**
- * Displays a pretty-printed program counter without instructionID
- * @return Current pc
- */
- public String toStringWithoutInstructionID() {
- StringBuilder sb = new StringBuilder();
- sb.append(DMLProgram.constructFunctionKey(this.namespace, this.fname));
- // sb.append(" instID ");
- // sb.append(this.instID);
- sb.append(": (line ");
- sb.append(this.lineNumber);
- sb.append(")");
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DebugState.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DebugState.java b/src/main/java/com/ibm/bi/dml/debug/DebugState.java
deleted file mode 100644
index b7330b9..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DebugState.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.Stack;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-
-public class DebugState
-{
-
- public String [] dmlScript;
- public Stack<DMLFrame> callStack = new Stack<DMLFrame>();
- public DMLProgramCounter pc = null, prevPC = null;
- public LocalVariableMap frameVariables=null;
- public String dbCommand=null;
- public String dbCommandArg=null;
- public boolean suspend = false;
- public volatile boolean nextCommand = false;
-
- private static DebugState instance = null;
-
- static {
- instance = new DebugState();
- }
-
- protected DebugState() {
- // So that one cannot instantiate DebugState and violate singleton pattern
- }
-
- public static DebugState getInstance() {
- return instance;
- }
-
- /////////////////////////////////
- // DML debugger public methods //
- /////////////////////////////////
-
- /**
- * Getter for current frame's program counter
- * @return Current frame program counter
- * @throws DMLRuntimeException
- */
- public DMLProgramCounter getPC() throws DMLRuntimeException {
- if(!DMLScript.ENABLE_DEBUG_MODE) {
- System.err.println("Error: This functionality (getPC) is available only in debug mode");
- //// Fatal error to avoid unintentional bugs
- throw new DMLRuntimeException("Error: This functionality (getPC) is available only in debug mode");
- }
- return pc;
- }
-
- /**
- * Getter for current frame's local variables
- * @return Current frame local variables
- */
- public LocalVariableMap getVariables() {
- return frameVariables;
- }
-
- /**
- * Getter for current frame
- * @return Current frame
- */
- public DMLFrame getCurrentFrame() {
- DMLFrame frame = new DMLFrame(frameVariables, pc);
- return frame;
- }
-
- /**
- * Setter for current frame's local variables
- * @param Current frame local variables
- */
- public void setVariables(LocalVariableMap vars) {
- frameVariables = vars;
- }
-
- /**
- * Is runtime ready to accept next command
- * @return true if the user interface can accept next command
- */
- public boolean canAcceptNextCommand() {
- return nextCommand;
- }
-
- /**
- * Set DML script field
- * @param lines DML script lines of code
- */
- public void setDMLScript(String [] lines) {
- dmlScript = lines;
- }
-
- /**
- * Print DML script source line
- * @param lineNum DML script line number
- */
- public void printDMLSourceLine(int lineNum) {
- if (lineNum > 0 && lineNum < dmlScript.length)
- System.out.format("%-4d %s\n",lineNum, dmlScript[lineNum-1]);
- }
-
- /**
- * Set debugger command into current runtime
- * @param command Debugger command
- */
- public void setCommand(String command) {
- this.dbCommand = command;
- }
-
- /**
- * Set debugger command argument into current runtime
- * @param cmdArg Debugger command argument
- */
- public void setCommandArg(String cmdArg) {
- this.dbCommandArg = cmdArg;
- }
-
- /**
- * Put current frame into stack due to function call
- * @param vars Caller's frame symbol table
- * @param pc Caller's frame program counter
- */
- public void pushFrame(LocalVariableMap vars, DMLProgramCounter pc) {
- callStack.push(new DMLFrame(vars, pc));
- }
-
- /**
- * Pop frame from stack as function call is done
- * @return Callee's frame (before function call)
- */
- public DMLFrame popFrame() {
- if (callStack.isEmpty())
- return null;
- return callStack.pop();
- }
-
- /**
- * Get stack frame at indicated location (if any)
- * @param location Frame position in call stack
- * @return Stack frame at specified location
- */
- protected DMLFrame getFrame(int location) {
- if (location < 0 || location >= callStack.size()) {
- return null;
- }
- return callStack.elementAt(location);
- }
-
- /**
- * Get current call stack (if any)
- * @return Stack callStack
- */
- public Stack<DMLFrame> getCallStack() {
- if (callStack.isEmpty())
- return null;
- return callStack;
- }
-
- /**
- * Display a full DML stack trace for a runtime exception
- */
- public void getDMLStackTrace(Exception e) {
- System.err.format("Runtime exception raised %s\n", e.toString());
- System.err.println("\t at " + this.pc.toString());
- if (this.callStack != null) {
- for (int i = callStack.size(); i > 0; i--) {
- DMLFrame frame = callStack.get(i-1);
- System.err.println("\t at " + frame.getPC().toString());
- }
- }
- }
-}
[13/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/PyDMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/PyDMLParserWrapper.java b/src/main/java/com/ibm/bi/dml/parser/python/PyDMLParserWrapper.java
deleted file mode 100644
index 69fb746..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/PyDMLParserWrapper.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.python;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ANTLRInputStream;
-import org.antlr.v4.runtime.BailErrorStrategy;
-import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.DefaultErrorStrategy;
-import org.antlr.v4.runtime.atn.PredictionMode;
-import org.antlr.v4.runtime.misc.ParseCancellationException;
-import org.antlr.v4.runtime.tree.ParseTree;
-import org.antlr.v4.runtime.tree.ParseTreeWalker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.antlr4.DMLParserWrapper;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PmlprogramContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StatementContext;
-import com.ibm.bi.dml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-
-/**
- * Logic of this wrapper is similar to DMLParserWrapper.
- *
- * Note: ExpressionInfo and StatementInfo are simply wrapper objects and are reused in both DML and PyDML parsers.
- *
- */
-public class PyDMLParserWrapper extends AParserWrapper
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- /**
- * Custom wrapper to convert statement into statement blocks. Called by doParse and in PydmlSyntacticValidator for for, parfor, while, ...
- * @param current a statement
- * @return corresponding statement block
- */
- public static StatementBlock getStatementBlock(com.ibm.bi.dml.parser.Statement current) {
- StatementBlock blk = null;
- if(current instanceof ParForStatement) {
- blk = new ParForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof ForStatement) {
- blk = new ForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof IfStatement) {
- blk = new IfStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof WhileStatement) {
- blk = new WhileStatementBlock();
- blk.addStatement(current);
- }
- else {
- // This includes ImportStatement
- blk = new StatementBlock();
- blk.addStatement(current);
- }
- return blk;
- }
-
- /**
- * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
- * @param fileName either full path or null --> only used for better error handling
- * @param dmlScript required
- * @param argVals
- * @return
- * @throws ParseException
- */
- @Override
- public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram prog = null;
-
- if(dmlScript == null || dmlScript.trim().isEmpty()) {
- throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
- }
-
- // Set the pipeline required for ANTLR parsing
- PyDMLParserWrapper parser = new PyDMLParserWrapper();
- prog = parser.doParse(fileName, dmlScript, argVals);
-
- if(prog == null) {
- throw new ParseException("One or more errors found during parsing. (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
- }
- return prog;
- }
-
- /**
- * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
- * @param fileName
- * @return null if atleast one error
- */
- public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram dmlPgm = null;
-
- ANTLRInputStream in;
- try {
- if(dmlScript == null) {
- dmlScript = DMLParserWrapper.readDMLScript(fileName);
- }
-
- InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
- in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
-// else {
-// if(!(new File(fileName)).exists()) {
-// throw new ParseException("ERROR: Cannot open file:" + fileName);
-// }
-// in = new ANTLRInputStream(new FileInputStream(fileName));
-// }
- } catch (FileNotFoundException e) {
- throw new ParseException("ERROR: Cannot find file:" + fileName);
- } catch (IOException e) {
- throw new ParseException("ERROR: Cannot open file:" + fileName);
- } catch (LanguageException e) {
- throw new ParseException("ERROR: " + e.getMessage());
- }
-
- PmlprogramContext ast = null;
- CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
-
- try {
- PydmlLexer lexer = new PydmlLexer(in);
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- PydmlParser antlr4Parser = new PydmlParser(tokens);
-
- boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
-
- if(tryOptimizedParsing) {
- // Try faster and simpler SLL
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
- antlr4Parser.removeErrorListeners();
- antlr4Parser.setErrorHandler(new BailErrorStrategy());
- try{
- ast = antlr4Parser.pmlprogram();
- // If successful, no need to try out full LL(*) ... SLL was enough
- }
- catch(ParseCancellationException ex) {
- // Error occurred, so now try full LL(*) for better error messages
- tokens.reset();
- antlr4Parser.reset();
- if(fileName != null) {
- errorListener.pushCurrentFileName(fileName);
- }
- else {
- errorListener.pushCurrentFileName("MAIN_SCRIPT");
- }
- // Set our custom error listener
- antlr4Parser.addErrorListener(errorListener);
- antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
- ast = antlr4Parser.pmlprogram();
- }
- }
- else {
- // Set our custom error listener
- antlr4Parser.removeErrorListeners();
- antlr4Parser.addErrorListener(errorListener);
- errorListener.pushCurrentFileName(fileName);
-
- // Now do the parsing
- ast = antlr4Parser.pmlprogram();
- }
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot parse the program:" + fileName);
- }
-
-
- try {
- // Now convert the parse tree into DMLProgram
- // Do syntactic validation while converting
- ParseTree tree = ast;
- // And also do syntactic validation
- ParseTreeWalker walker = new ParseTreeWalker();
- PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener);
- PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals);
- walker.walk(validator, tree);
- errorListener.popFileName();
- if(errorListener.isAtleastOneError()) {
- return null;
- }
- dmlPgm = createDMLProgram(ast);
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram:" + e.getMessage());
- }
-
- return dmlPgm;
- }
-
- private DMLProgram createDMLProgram(PmlprogramContext ast) {
-
- DMLProgram dmlPgm = new DMLProgram();
-
- // First add all the functions
- for(FunctionStatementContext fn : ast.functionBlocks) {
- FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
- functionStmtBlk.addStatement(fn.info.stmt);
- try {
- // TODO: currently the logic of nested namespace is not clear.
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
- } catch (LanguageException e) {
- LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
- return null;
- }
- }
-
- // Then add all the statements
- for(StatementContext stmtCtx : ast.blocks) {
- Statement current = stmtCtx.info.stmt;
- if(current == null) {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
- return null;
- }
-
- // Ignore Newline logic
- if(current.isEmptyNewLineStatement()) {
- continue;
- }
-
- if(current instanceof ImportStatement) {
- // Handle import statements separately
- if(stmtCtx.info.namespaces != null) {
- // Add the DMLProgram entries into current program
- for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
- dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-// // Don't add DMLProgram into the current program, just add function statements
-// // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-// // Add function statements to current dml program
-// DMLProgram importedPgm = entry.getValue();
-//
-// try {
-// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
-// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
-// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
-// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
-// }
-// else {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
-// return null;
-// }
-// }
-// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
-// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
-// }
-// } catch (LanguageException e) {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement");
-// return null;
-// }
- }
- }
- else {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
- return null;
- }
- }
-
- // Now wrap statement into individual statement block
- // merge statement will take care of merging these blocks
- dmlPgm.addStatementBlock(getStatementBlock(current));
- }
-
- dmlPgm.mergeStatementBlocks();
- return dmlPgm;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/Pydml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/Pydml.g4 b/src/main/java/com/ibm/bi/dml/parser/python/Pydml.g4
deleted file mode 100644
index 821ca29..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/Pydml.g4
+++ /dev/null
@@ -1,382 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-grammar Pydml;
-
-@header
-{
- // package com.ibm.bi.dml.python;
- //import com.ibm.bi.dml.parser.antlr4.StatementInfo;
- //import com.ibm.bi.dml.parser.antlr4.ExpressionInfo;
-}
-
-// This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html
-
-tokens { INDENT, DEDENT }
-
-@lexer::members {
- private boolean debugIndentRules = false;
-
- // Indentation level stack
- private java.util.Stack<Integer> indents = new java.util.Stack<Integer>();
-
- // Extra tokens queue (see the NEWLINE rule).
- private java.util.Queue<Token> tokens = new java.util.LinkedList<Token>();
-
- // Number of opened braces, brackets and parenthesis.
- private int opened = 0;
-
- // This is only used to set the line number for dedent
- private Token lastToken = null;
-
-
- @Override
- public void emit(Token t) {
- if(debugIndentRules)
- System.out.println("Emitted token:" + t);
-
- super.setToken(t);
- tokens.offer(t);
- }
-
-
- @Override
- public Token nextToken() {
- if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
- if(debugIndentRules)
- System.out.println("EOF reached and expecting some DEDENTS, so emitting them");
-
- tokens.poll();
- this.emit(commonToken(PydmlParser.NEWLINE, "\n"));
-
- // Now emit as much DEDENT tokens as needed.
- while (!indents.isEmpty()) {
- if(debugIndentRules)
- System.out.println("Emitting (inserted) DEDENTS");
-
- this.emit(createDedent());
- indents.pop();
- }
- // Put the EOF back on the token stream.
- this.emit(commonToken(PydmlParser.EOF, "<EOF>"));
- }
- Token next = super.nextToken();
- if (next.getChannel() == Token.DEFAULT_CHANNEL) {
- // Keep track of the last token on the default channel.
- this.lastToken = next;
- }
- Token retVal = tokens.isEmpty() ? next : tokens.poll();
-
- if(debugIndentRules)
- System.out.println("Returning nextToken: [" + retVal + "]<<" + tokens.isEmpty());
-
- return retVal;
- }
-
- private Token createDedent() {
- CommonToken dedent = commonToken(PydmlParser.DEDENT, "");
- dedent.setLine(this.lastToken.getLine());
- return dedent;
- }
-
- private CommonToken commonToken(int type, String text) {
- // Nike: Main change: This logic was screwed up and was emitting additional 3 characters, so commenting it for now.
- // int start = this.getCharIndex();
- // int stop = start + text.length();
- // return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop);
- return new CommonToken(type, text); // Main change
- }
-
- // Calculates the indentation level from the spaces:
- // "Tabs are replaced (from left to right) by one to eight spaces
- // such that the total number of characters up to and including
- // the replacement is a multiple of eight [...]"
- // https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
- static int getIndentationCount(String spaces) {
- int count = 0;
- for (char ch : spaces.toCharArray()) {
- switch (ch) {
- case '\t':
- count += 8 - (count % 8);
- break;
- default:
- // A normal space char.
- count++;
- }
- }
- return count;
- }
-}
-
-
-// 2. Modify this g4 by comparing it with Java:
-// - https://pythonconquerstheuniverse.wordpress.com/2009/10/03/python-java-a-side-by-side-comparison/
-// - http://www.cs.gordon.edu/courses/cps122/handouts-2014/From%20Python%20to%20Java%20Lecture/A%20Comparison%20of%20the%20Syntax%20of%20Python%20and%20Java.pdf
-// - http://cs.joensuu.fi/~pviktor/python/slides/cheatsheet.pdf
-// - http://www.interfaceware.com/manual/chameleon/scripts/quickreference.pdf
-
-// DML Program is a list of expression
-// For now, we only allow global function definitions (not nested or inside a while block)
-pmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF;
-
-
-
-statement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // ImportStatement
- 'source' OPEN_PAREN filePath = STRING CLOSE_PAREN 'as' namespace=ID NEWLINE # ImportStatement
- | 'setwd' OPEN_PAREN pathValue = STRING CLOSE_PAREN NEWLINE # PathStatement
- // ------------------------------------------
- // AssignmentStatement
- | targetList+=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement
- // ------------------------------------------
- // Treat function call as AssignmentStatement or MultiAssignmentStatement
- // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
- // Convert FunctionCallIdentifier(paramExprs, ..) -> source
- | // TODO: Throw an informative error if user doesnot provide the optional assignment
- ( targetList+=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement
- | OPEN_BRACK targetList+=dataIdentifier (',' targetList+=dataIdentifier)* CLOSE_BRACK '=' name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallMultiAssignmentStatement
- // {notifyErrorListeners("Too many parentheses");}
- // We don't support block statement
- // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
- // ------------------------------------------
- | targetList+=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement
- // IfStatement
- // | 'if' OPEN_PAREN predicate=expression CLOSE_PAREN (ifBody+=statement ';'* | NEWLINE INDENT (ifBody+=statement)+ DEDENT ) ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
- | 'if' (OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression) ':' NEWLINE INDENT (ifBody+=statement)+ DEDENT ('else' ':' NEWLINE INDENT (elseBody+=statement)+ DEDENT )? # IfStatement
- // ------------------------------------------
- // ForStatement & ParForStatement
- | 'for' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ForStatement
- // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
- | 'parfor' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ParForStatement
- | 'while' ( OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # WhileStatement
- // ------------------------------------------
- | NEWLINE #IgnoreNewLine
-;
-
-iterablePredicate returns [ ExpressionInfo info ]
- @init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- } :
- from=expression ':' to=expression #IterablePredicateColonExpression
- | ID OPEN_PAREN from=expression ',' to=expression ',' increment=expression CLOSE_PAREN #IterablePredicateSeqExpression
- ;
-
-functionStatement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // FunctionStatement & ExternalFunctionStatement
- // small change: only allow typed arguments here ... instead of data identifier
- 'def' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? ':' NEWLINE INDENT (body+=statement)+ DEDENT # InternalFunctionDefExpression
- | 'defExternal' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? 'implemented' 'in' OPEN_PAREN ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? CLOSE_PAREN NEWLINE # ExternalFunctionDefExpression
- // ------------------------------------------
-;
-
-
-// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
-dataIdentifier returns [ ExpressionInfo dataInfo ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new com.ibm.bi.dml.parser.DataIdentifier();
-} :
- // ------------------------------------------
- // IndexedIdentifier
- name=ID OPEN_BRACK (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? CLOSE_BRACK # IndexedExpression
- // ------------------------------------------
- | ID # SimpleDataIdentifierExpression
- | COMMANDLINE_NAMED_ID # CommandlineParamExpression
- | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
-;
-expression returns [ ExpressionInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- // $info.expr = new com.ibm.bi.dml.parser.BinaryExpression(com.ibm.bi.dml.parser.Expression.BinaryOp.INVALID);
-} :
- // ------------------------------------------
- // BinaryExpression
- // power
- <assoc=right> left=expression op='**' right=expression # PowerExpression
- // unary plus and minus
- | op=('-'|'+') left=expression # UnaryExpression
- // sequence - since we are only using this into for loop => Array not supported
- //| left=expression op=':' right=expression # SequenceExpression
- // matrix multiply
- // | left=expression op='*' right=expression # MatrixMulExpression
- // modulus and integer division
- | left=expression op=('//' | '%' ) right=expression # ModIntDivExpression
- // arithmetic multiply and divide
- | left=expression op=('*'|'/') right=expression # MultDivExpression
- // arithmetic addition and subtraction
- | left=expression op=('+'|'-') right=expression # AddSubExpression
- // ------------------------------------------
- // RelationalExpression
- | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
- // ------------------------------------------
- // BooleanExpression
- // boolean not
- | op='!' left=expression # BooleanNotExpression
- // boolean and
- | left=expression op=('&'|'and') right=expression # BooleanAndExpression
- // boolean or
- | left=expression op=('|'|'or') right=expression # BooleanOrExpression
-
- // ---------------------------------
- // only applicable for builtin function expressions
- // Add following additional functions and check number of parameters:
- // power, full, matrix, reshape, dot
- // Also take care whether there is y.transpose() => which sometinamespace
- | name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN ';'* # BuiltinFunctionExpression
-
- // 4. Atomic
- | OPEN_PAREN left=expression CLOSE_PAREN # AtomicExpression
-
- // Should you allow indexed expression here ?
- // | OPEN_BRACK targetList+=expression (',' targetList+=expression)* CLOSE_BRACK # MultiIdExpression
-
- // | BOOLEAN # ConstBooleanIdExpression
- | 'True' # ConstTrueExpression
- | 'False' # ConstFalseExpression
- | INT # ConstIntIdExpression
- | DOUBLE # ConstDoubleIdExpression
- | STRING # ConstStringIdExpression
- | dataIdentifier # DataIdExpression
- // Special
- // | 'NULL' | 'NA' | 'Inf' | 'NaN'
-;
-
-typedArgNoAssign : paramName=ID ':' paramType=ml_type ;
-parameterizedExpression : (paramName=ID '=')? paramVal=expression;
-strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
-strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
-// sometimes this is matrix object and sometimes its namespace
-ID : (ALPHABET (ALPHABET|DIGIT|'_')* '.')? ALPHABET (ALPHABET|DIGIT|'_')*
- // Special ID cases:
- // | 'matrix' // --> This is a special case which causes lot of headache
- // | 'scalar' | 'float' | 'int' | 'bool' // corresponds to as.scalar, as.double, as.integer and as.logical
- | 'index.return'
-;
-// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
-// Therefore, ugly work around for checking datatype
-ml_type : valueType | dataType OPEN_BRACK valueType CLOSE_BRACK;
-// Note to reduce number of keywords, these are case-sensitive,
-// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
-valueType:
- ID # ValueDataTypeCheck
- // 'int' | 'str' | 'bool' | 'float'
-;
-dataType:
- // 'scalar' # ScalarDataTypeDummyCheck
- // |
- ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
- //| 'matrix' //---> See ID, this causes lot of headache
- ;
-INT : DIGIT+ [Ll]?;
-// BOOLEAN : 'TRUE' | 'FALSE';
-DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
-| DIGIT+ EXP? [Ll]?
-| '.' DIGIT+ EXP? [Ll]?
-;
-DIGIT: '0'..'9';
-ALPHABET : [a-zA-Z] ;
-fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
-COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
-COMMANDLINE_POSITION_ID: '$' DIGIT+;
-
-// supports single and double quoted string with escape characters
-STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
-fragment ESC : '\\' [abtnfrv"'\\] ;
-// Comments, whitespaces and new line
-// LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
-// MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
-// WHITESPACE : (' ' | '\r' | '\n')+ -> skip ;
-
-OPEN_BRACK : '[' {opened++;};
-CLOSE_BRACK : ']' {opened--;};
-OPEN_PAREN : '(' {opened++;};
-CLOSE_PAREN : ')' {opened--;};
-// OPEN_BRACE : '{' {opened++;};
-// CLOSE_BRACE : '}' {opened--;};
-
-fragment SPACES : [ \t]+ ;
-fragment COMMENT : '#' ~[\r\n]* ;
-fragment LINE_JOINING : '\\' SPACES? ( '\r'? '\n' | '\r' ) ;
-
-NEWLINE : ( '\r'? '\n' | '\r' ) SPACES?
-{
- String newLine = getText().replaceAll("[^\r\n]+", "");
- String spaces = getText().replaceAll("[\r\n]+", "");
- int next = _input.LA(1);
- if (opened > 0 || next == '\r' || next == '\n' || next == '#') {
- // If we're inside a list or on a blank line, ignore all indents,
- // dedents and line breaks.
- skip();
- if(debugIndentRules) {
- if(next == '\r' || next == '\n') {
- System.out.println("4.1 Skipping (blank lines)");
- }
- else if(next == '#') {
- System.out.println("4.2 Skipping (comment)");
- }
- else {
- System.out.println("4.2 Skipping something else");
- }
- }
- }
- else {
- emit(commonToken(NEWLINE, newLine));
-
- int indent = getIndentationCount(spaces);
- int previous = indents.isEmpty() ? 0 : indents.peek();
- if (indent == previous) {
- if(debugIndentRules)
- System.out.println("3. Skipping identation as of same size:" + next);
-
- // skip indents of the same size as the present indent-size
- skip();
- }
- else if (indent > previous) {
- if(debugIndentRules)
- System.out.println("1. Indent:" + next);
-
- indents.push(indent);
- emit(commonToken(PydmlParser.INDENT, spaces));
- }
- else {
- // Possibly emit more than 1 DEDENT token.
- while(!indents.isEmpty() && indents.peek() > indent) {
- if(debugIndentRules)
- System.out.println("2. Dedent:" + next);
-
- this.emit(createDedent());
- indents.pop();
- }
- }
- }
-}
-;
-
-SKIP : ( SPACES | COMMENT | LINE_JOINING ) -> skip ;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticErrorListener.java b/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticErrorListener.java
deleted file mode 100644
index 8a29038..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticErrorListener.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.python;
-
-import java.util.Stack;
-
-import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Recognizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-
-public class PydmlSyntacticErrorListener {
-
-
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static class CustomDmlErrorListener extends BaseErrorListener {
-
- private boolean atleastOneError = false;
- private Stack<String> currentFileName = new Stack<String>();
-
- public void pushCurrentFileName(String currentFilePath) {
- currentFileName.push(currentFilePath);
- }
-
- public String peekFileName() {
- return currentFileName.peek();
- }
-
- public String popFileName() {
- return currentFileName.pop();
- }
-
- public void validationError(int line, int charPositionInLine, String msg) {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty()) {
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- }
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public void validationWarning(int line, int charPositionInLine, String msg) {
- try {
- //atleastOneError = true; ---> not an error, just warning
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.warn("ERROR: while customizing error message:" + e1);
- }
- }
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
- int line, int charPositionInLine,
- String msg, RecognitionException e)
- {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public boolean isAtleastOneError() {
- return atleastOneError;
- }
-
- public void setAtleastOneError(boolean atleastOneError) {
- this.atleastOneError = atleastOneError;
- }
- }
-}
[34/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java b/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
deleted file mode 100644
index b8e894e..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
+++ /dev/null
@@ -1,2126 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.recompile;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.wink.json4j.JSONObject;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.lops.CSVReBlock;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTreeConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Explain.ExplainType;
-import com.ibm.bi.dml.utils.JSONHelper;
-
-/**
- * Dynamic recompilation of hop dags to runtime instructions, which includes the
- * following substeps:
- *
- * (1) deep copy hop dag, (2) refresh matrix characteristics, (3) apply
- * dynamic rewrites, (4) refresh memory estimates, (5) construct lops (incl
- * operator selection), and (6) generate runtime program (incl piggybacking).
- *
- *
- */
-public class Recompiler
-{
-
- private static final Log LOG = LogFactory.getLog(Recompiler.class.getName());
-
- //Max threshold for in-memory reblock of text input [in bytes]
- //reason: single-threaded text read at 20MB/s, 1GB input -> 50s (should exploit parallelism)
- //note that we scale this threshold up by the degree of available parallelism
- private static final long CP_REBLOCK_THRESHOLD_SIZE = (long)1024*1024*1024;
- private static final long CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE = (long)256*1024*1024;
- private static final long CP_TRANSFORM_UNKNOWN_THRESHOLD_SIZE = (long)1024*1024*1024;
-
- //reused rewriter for dynamic rewrites during recompile
- private static ProgramRewriter rewriter = new ProgramRewriter(false, true);
-
- /**
- * Re-initializes the recompiler according to the current optimizer flags.
- */
- public static void reinitRecompiler()
- {
- rewriter = new ProgramRewriter(false, true);
- }
-
- /**
- * A) Recompile basic program block hop DAG.
- *
- * We support to basic types inplace or via deep copy. Deep copy is the default and is required
- * in order to apply non-reversible rewrites. In-place is required in order to modify the existing
- * hops (e.g., for parfor pre-recompilation).
- *
- * @param hops
- * @param vars
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDag( StatementBlock sb, ArrayList<Hop> hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, long tid )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- //however, we create deep copies for most dags to allow for concurrent recompile
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // prepare hops dag for recompile
- if( !inplace ){
- // deep copy hop dag (for non-reversable rewrites)
- hops = deepCopyHopsDag(hops);
- }
- else {
- // clear existing lops
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rClearLops( hopRoot );
- }
-
- // replace scalar reads with literals
- if( !inplace ) {
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rReplaceLiterals( hopRoot, vars );
- }
-
- // refresh matrix characteristics (update stats)
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rUpdateStatistics( hopRoot, vars );
-
- // dynamic hop rewrites
- if( !inplace )
- rewriter.rewriteHopDAGs( hops, null );
-
- // refresh memory estimates (based on updated stats,
- // before: init memo table with propagated worst-case estimates,
- // after: extract worst-case estimates from memo table
- Hop.resetVisitStatus(hops);
- MemoTable memo = new MemoTable();
- memo.init(hops, status);
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- hopRoot.refreshMemEstimates(memo);
- memo.extract(hops, status);
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- for( Hop hopRoot : hops ){
- Lop lops = hopRoot.constructLops();
- lops.addToDag(dag);
- }
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
- }
-
- // replace thread ids in new instructions
- if( tid != 0 ) //only in parfor context
- newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
-
- // explain recompiled hops / instructions
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS ){
- LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+"):\n" +
- Explain.explainHops(hops, 1));
- }
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME ){
- LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+"):\n" +
- Explain.explain(newInst, 1));
- }
-
- return newInst;
- }
-
- /**
- * B) Recompile predicate hop DAG (single root):
- *
- * Note: This overloaded method is required for predicate instructions because
- * they have only a single hops DAG and we need to synchronize on the original
- * (shared) hops object. Hence, we cannot create any wrapper arraylist for each
- * recompilation - this would result in race conditions for concurrent recompilation
- * in a parfor body.
- *
- * Note: no statementblock passed because for predicate dags we dont have separate live variable analysis information.
- *
- * @param hops
- * @param vars
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDag( Hop hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, long tid )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // prepare hops dag for recompile
- if( !inplace ) {
- // deep copy hop dag (for non-reversable rewrites)
- //(this also clears existing lops in the created dag)
- hops = deepCopyHopsDag(hops);
- }
- else {
- // clear existing lops
- hops.resetVisitStatus();
- rClearLops( hops );
- }
-
- // replace scalar reads with literals
- if( !inplace ) {
- hops.resetVisitStatus();
- rReplaceLiterals( hops, vars );
- }
-
- // refresh matrix characteristics (update stats)
- hops.resetVisitStatus();
- rUpdateStatistics( hops, vars );
-
- // dynamic hop rewrites
- if( !inplace )
- rewriter.rewriteHopDAG( hops, null );
-
- // refresh memory estimates (based on updated stats)
- MemoTable memo = new MemoTable();
- hops.resetVisitStatus();
- memo.init(hops, status);
- hops.resetVisitStatus();
- hops.refreshMemEstimates(memo);
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- Lop lops = hops.constructLops();
- lops.addToDag(dag);
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
- }
-
- // replace thread ids in new instructions
- if( tid != 0 ) //only in parfor context
- newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
-
- // explain recompiled instructions
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS )
- LOG.info("EXPLAIN RECOMPILE \nPRED (line "+hops.getBeginLine()+"):\n" + Explain.explain(hops,1));
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME )
- LOG.info("EXPLAIN RECOMPILE \nPRED (line "+hops.getBeginLine()+"):\n" + Explain.explain(newInst,1));
-
- return newInst;
- }
-
- /**
- * C) Recompile basic program block hop DAG, but forced to CP.
- *
- * This happens always 'inplace', without statistics updates, and
- * without dynamic rewrites.
- *
- * @param hops
- * @param tid
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDag2Forced( StatementBlock sb, ArrayList<Hop> hops, long tid, ExecType et )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- //however, we create deep copies for most dags to allow for concurrent recompile
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // clear existing lops
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rClearLops( hopRoot );
-
- // update exec type
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rSetExecType( hopRoot, et );
- Hop.resetVisitStatus(hops);
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- for( Hop hopRoot : hops ){
- Lop lops = hopRoot.constructLops();
- lops.addToDag(dag);
- }
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
- }
-
- // replace thread ids in new instructions
- if( tid != 0 ) //only in parfor context
- newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
-
- return newInst;
- }
-
- /**
- * D) Recompile predicate hop DAG (single root), but forced to CP.
- *
- * This happens always 'inplace', without statistics updates, and
- * without dynamic rewrites.
- *
- * @param hops
- * @param tid
- * @param et
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDag2Forced( Hop hops, long tid, ExecType et )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // clear existing lops
- hops.resetVisitStatus();
- rClearLops( hops );
-
- // update exec type
- hops.resetVisitStatus();
- rSetExecType( hops, et );
- hops.resetVisitStatus();
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- Lop lops = hops.constructLops();
- lops.addToDag(dag);
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
- }
-
- // replace thread ids in new instructions
- if( tid != 0 ) //only in parfor context
- newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
-
- return newInst;
- }
-
- /**
- *
- * @param sb
- * @param hops
- * @return
- * @throws HopsException
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDagInstructions( StatementBlock sb, ArrayList<Hop> hops )
- throws HopsException, LopsException, DMLRuntimeException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- //however, we create deep copies for most dags to allow for concurrent recompile
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // clear existing lops
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rClearLops( hopRoot );
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- for( Hop hopRoot : hops ){
- Lop lops = hopRoot.constructLops();
- lops.addToDag(dag);
- }
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
- }
-
- // explain recompiled hops / instructions
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS ){
- LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+"):\n" +
- Explain.explainHops(hops, 1));
- }
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME ){
- LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+"):\n" +
- Explain.explain(newInst, 1));
- }
-
- return newInst;
- }
-
- /**
- *
- * @param hops
- * @return
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static ArrayList<Instruction> recompileHopsDagInstructions( Hop hops )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- ArrayList<Instruction> newInst = null;
-
- //need for synchronization as we do temp changes in shared hops/lops
- synchronized( hops )
- {
- LOG.debug ("\n**************** Optimizer (Recompile) *************\nMemory Budget = " +
- OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
-
- // clear existing lops
- hops.resetVisitStatus();
- rClearLops( hops );
-
- // construct lops
- Dag<Lop> dag = new Dag<Lop>();
- Lop lops = hops.constructLops();
- lops.addToDag(dag);
-
- // generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
- }
-
- // explain recompiled instructions
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS )
- LOG.info("EXPLAIN RECOMPILE \nPRED (line "+hops.getBeginLine()+"):\n" + Explain.explain(hops,1));
- if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME )
- LOG.info("EXPLAIN RECOMPILE \nPRED (line "+hops.getBeginLine()+"):\n" + Explain.explain(newInst,1));
-
- return newInst;
- }
-
-
- /**
- *
- * @param pbs
- * @param vars
- * @param tid
- * @throws DMLRuntimeException
- */
- public static void recompileProgramBlockHierarchy( ArrayList<ProgramBlock> pbs, LocalVariableMap vars, long tid, boolean resetRecompile )
- throws DMLRuntimeException
- {
- try
- {
- RecompileStatus status = new RecompileStatus();
-
- synchronized( pbs )
- {
- for( ProgramBlock pb : pbs )
- rRecompileProgramBlock(pb, vars, status, tid, resetRecompile);
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to recompile program block hierarchy.", ex);
- }
- }
-
- /**
- * Method to recompile program block hierarchy to forced execution time. This affects also
- * referenced functions and chains of functions. Use et==null in order to release the forced
- * exec type.
- *
- * @param pbs
- * @param tid
- * @throws DMLRuntimeException
- */
- public static void recompileProgramBlockHierarchy2Forced( ArrayList<ProgramBlock> pbs, long tid, HashSet<String> fnStack, ExecType et )
- throws DMLRuntimeException
- {
- try
- {
- synchronized( pbs )
- {
- for( ProgramBlock pb : pbs )
- rRecompileProgramBlock2Forced(pb, tid, fnStack, et);
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to recompile program block hierarchy to CP.", ex);
- }
- }
-
- /**
- * This method does NO full program block recompile (no stats update, no rewrites, no recursion) but
- * only regenerates lops and instructions. The primary use case is recompilation after are hop configuration
- * changes which allows to preserve statistics (e.g., propagated worst case stats from other program blocks)
- * and better performance for recompiling individual program blocks.
- *
- * @param pb
- * @throws IOException
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws LopsException
- * @throws HopsException
- */
- public static void recompileProgramBlockInstructions(ProgramBlock pb)
- throws HopsException, LopsException, DMLRuntimeException, DMLUnsupportedOperationException, IOException
- {
- if( pb instanceof WhileProgramBlock )
- {
- //recompile while predicate instructions
- WhileProgramBlock wpb = (WhileProgramBlock)pb;
- WhileStatementBlock wsb = (WhileStatementBlock) pb.getStatementBlock();
- if( wsb!=null && wsb.getPredicateHops()!=null )
- wpb.setPredicate(recompileHopsDagInstructions(wsb.getPredicateHops()));
- }
- else if( pb instanceof IfProgramBlock )
- {
- //recompile if predicate instructions
- IfProgramBlock ipb = (IfProgramBlock)pb;
- IfStatementBlock isb = (IfStatementBlock) pb.getStatementBlock();
- if( isb!=null && isb.getPredicateHops()!=null )
- ipb.setPredicate(recompileHopsDagInstructions(isb.getPredicateHops()));
- }
- else if( pb instanceof ForProgramBlock )
- {
- //recompile for/parfor predicate instructions
- ForProgramBlock fpb = (ForProgramBlock)pb;
- ForStatementBlock fsb = (ForStatementBlock) pb.getStatementBlock();
- if( fsb!=null && fsb.getFromHops()!=null )
- fpb.setFromInstructions(recompileHopsDagInstructions(fsb.getFromHops()));
- if( fsb!=null && fsb.getToHops()!=null )
- fpb.setToInstructions(recompileHopsDagInstructions(fsb.getToHops()));
- if( fsb!=null && fsb.getIncrementHops()!=null )
- fpb.setIncrementInstructions(recompileHopsDagInstructions(fsb.getIncrementHops()));
- }
- else
- {
- //recompile last-level program block instructions
- StatementBlock sb = pb.getStatementBlock();
- if( sb!=null && sb.get_hops()!=null ) {
- pb.setInstructions(recompileHopsDagInstructions(sb, sb.get_hops()));
- }
- }
- }
-
- /**
- *
- * @param hops
- * @return
- */
- public static boolean requiresRecompilation( ArrayList<Hop> hops )
- {
- boolean ret = false;
-
- if( hops != null )
- {
- synchronized( hops )
- {
- Hop.resetVisitStatus(hops);
- for( Hop hop : hops )
- {
- ret |= rRequiresRecompile(hop);
- if( ret ) break; // early abort
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param hops
- * @return
- */
- public static boolean requiresRecompilation( Hop hop )
- {
- boolean ret = false;
-
- if( hop != null )
- {
- synchronized( hop )
- {
- hop.resetVisitStatus();
- ret = rRequiresRecompile(hop);
- }
- }
-
- return ret;
- }
-
-
- /**
- * Deep copy of hops dags for parallel recompilation.
- *
- * @param hops
- * @return
- * @throws CloneNotSupportedException
- */
- public static ArrayList<Hop> deepCopyHopsDag( ArrayList<Hop> hops )
- throws HopsException
- {
- ArrayList<Hop> ret = new ArrayList<Hop>();
-
- try {
- //note: need memo table over all independent DAGs in order to
- //account for shared transient reads (otherwise more instructions generated)
- HashMap<Long, Hop> memo = new HashMap<Long, Hop>(); //orig ID, new clone
- for( Hop hopRoot : hops )
- ret.add(rDeepCopyHopsDag(hopRoot, memo));
- }
- catch(Exception ex)
- {
- throw new HopsException(ex);
- }
-
- return ret;
- }
-
- /**
- * Deep copy of hops dags for parallel recompilation.
- *
- * @param hops
- * @return
- * @throws CloneNotSupportedException
- */
- public static Hop deepCopyHopsDag( Hop hops )
- throws HopsException
- {
- Hop ret = null;
-
- try {
- HashMap<Long, Hop> memo = new HashMap<Long, Hop>(); //orig ID, new clone
- ret = rDeepCopyHopsDag(hops, memo);
- }
- catch(Exception ex)
- {
- throw new HopsException(ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @param hops
- * @param memo
- * @return
- * @throws CloneNotSupportedException
- */
- private static Hop rDeepCopyHopsDag( Hop hops, HashMap<Long,Hop> memo )
- throws CloneNotSupportedException
- {
- Hop ret = memo.get(hops.getHopID());
-
- //create clone if required
- if( ret == null )
- {
- ret = (Hop) hops.clone();
- ArrayList<Hop> tmp = new ArrayList<Hop>();
-
- //create new childs
- for( Hop in : hops.getInput() )
- {
- Hop newIn = rDeepCopyHopsDag(in, memo);
- tmp.add(newIn);
- }
- //modify references of childs
- for( Hop in : tmp )
- {
- ret.getInput().add(in);
- in.getParent().add(ret);
- }
-
- memo.put(hops.getHopID(), ret);
- }
-
- return ret;
- }
-
-
- public static void updateFunctionNames(ArrayList<Hop> hops, long pid)
- {
- Hop.resetVisitStatus(hops);
- for( Hop hopRoot : hops )
- rUpdateFunctionNames( hopRoot, pid );
- }
-
- public static void rUpdateFunctionNames( Hop hop, long pid )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //update function names
- if( hop instanceof FunctionOp && ((FunctionOp)hop).getFunctionType() != FunctionType.MULTIRETURN_BUILTIN) {
- FunctionOp fop = (FunctionOp) hop;
- fop.setFunctionName( fop.getFunctionName() +
- ProgramConverter.CP_CHILD_THREAD + pid);
- }
-
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- rUpdateFunctionNames(c, pid);
-
- hop.setVisited(VisitStatus.DONE);
- }
-
-
- //////////////////////////////
- // private helper functions //
- //////////////////////////////
-
-
- /**
- *
- * @param pb
- * @param vars
- * @param tid
- * @throws IOException
- * @throws DMLUnsupportedOperationException
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws HopsException
- */
- private static void rRecompileProgramBlock( ProgramBlock pb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
- throws HopsException, DMLRuntimeException, LopsException, DMLUnsupportedOperationException, IOException
- {
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock)pb;
- WhileStatementBlock wsb = (WhileStatementBlock) wpb.getStatementBlock();
- //recompile predicate
- recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
- //remove updated scalars because in loop
- removeUpdatedScalars(vars, wsb);
- //copy vars for later compare
- LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
- RecompileStatus oldStatus = (RecompileStatus) status.clone();
- for (ProgramBlock pb2 : wpb.getChildBlocks())
- rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
- if( reconcileUpdatedCallVarsLoops(oldVars, vars, wsb)
- | reconcileUpdatedCallVarsLoops(oldStatus, status, wsb) ) {
- //second pass with unknowns if required
- recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
- for (ProgramBlock pb2 : wpb.getChildBlocks())
- rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
- }
- removeUpdatedScalars(vars, wsb);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock)pb;
- IfStatementBlock isb = (IfStatementBlock)ipb.getStatementBlock();
- //recompile predicate
- recompileIfPredicate(ipb, isb, vars, status, tid, resetRecompile);
- //copy vars for later compare
- LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
- LocalVariableMap varsElse = (LocalVariableMap) vars.clone();
- RecompileStatus oldStatus = (RecompileStatus)status.clone();
- RecompileStatus statusElse = (RecompileStatus)status.clone();
- for( ProgramBlock pb2 : ipb.getChildBlocksIfBody() )
- rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
- for( ProgramBlock pb2 : ipb.getChildBlocksElseBody() )
- rRecompileProgramBlock(pb2, varsElse, statusElse, tid, resetRecompile);
- reconcileUpdatedCallVarsIf(oldVars, vars, varsElse, isb);
- reconcileUpdatedCallVarsIf(oldStatus, status, statusElse, isb);
- removeUpdatedScalars(vars, ipb.getStatementBlock());
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock fpb = (ForProgramBlock)pb;
- ForStatementBlock fsb = (ForStatementBlock) fpb.getStatementBlock();
- //recompile predicates
- recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
- //remove updated scalars because in loop
- removeUpdatedScalars(vars, fpb.getStatementBlock());
- //copy vars for later compare
- LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
- RecompileStatus oldStatus = (RecompileStatus) status.clone();
- for( ProgramBlock pb2 : fpb.getChildBlocks() )
- rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
- if( reconcileUpdatedCallVarsLoops(oldVars, vars, fsb)
- | reconcileUpdatedCallVarsLoops(oldStatus, status, fsb)) {
- //second pass with unknowns if required
- recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
- for( ProgramBlock pb2 : fpb.getChildBlocks() )
- rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
- }
- removeUpdatedScalars(vars, fpb.getStatementBlock());
- }
- else if ( pb instanceof FunctionProgramBlock ) //includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
- {
- //do nothing
- }
- else
- {
- StatementBlock sb = pb.getStatementBlock();
- ArrayList<Instruction> tmp = pb.getInstructions();
-
- if( sb != null //recompile all for stats propagation and recompile flags
- //&& Recompiler.requiresRecompilation( sb.get_hops() )
- /*&& !Recompiler.containsNonRecompileInstructions(tmp)*/ )
- {
- tmp = Recompiler.recompileHopsDag(sb, sb.get_hops(), vars, status, true, tid);
- pb.setInstructions( tmp );
-
- //propagate stats across hops (should be executed on clone of vars)
- Recompiler.extractDAGOutputStatistics(sb.get_hops(), vars);
-
- //reset recompilation flags (w/ special handling functions)
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && !containsRootFunctionOp(sb.get_hops())
- && resetRecompile )
- {
- Hop.resetRecompilationFlag(sb.get_hops(), ExecType.CP);
- sb.updateRecompilationFlag();
- }
- }
-
- }
-
- }
-
-
- /**
- *
- * @param oldCallVars
- * @param callVars
- * @param sb
- * @return
- */
- public static boolean reconcileUpdatedCallVarsLoops( LocalVariableMap oldCallVars, LocalVariableMap callVars, StatementBlock sb )
- {
- boolean requiresRecompile = false;
-
- //handle matrices
- for( String varname : sb.variablesUpdated().getVariableNames() )
- {
- Data dat1 = oldCallVars.get(varname);
- Data dat2 = callVars.get(varname);
- if( dat1!=null && dat1 instanceof MatrixObject && dat2!=null && dat2 instanceof MatrixObject )
- {
- MatrixObject moOld = (MatrixObject) dat1;
- MatrixObject mo = (MatrixObject) dat2;
- MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
- MatrixCharacteristics mc = mo.getMatrixCharacteristics();
-
- if( mcOld.getRows() != mc.getRows()
- || mcOld.getCols() != mc.getCols()
- || mcOld.getNonZeros() != mc.getNonZeros() )
- {
- long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
- //handle row dimension change in body
- if( mcOld.getRows() != mc.getRows() ) {
- ldim1=-1; //unknown
- requiresRecompile = true;
- }
- //handle column dimension change in body
- if( mcOld.getCols() != mc.getCols() ) {
- ldim2=-1; //unknown
- requiresRecompile = true;
- }
- //handle sparsity change
- if( mcOld.getNonZeros() != mc.getNonZeros() ) {
- lnnz=-1; //unknown
- requiresRecompile = true;
- }
-
- MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
- callVars.put(varname, moNew);
- }
- }
- }
-
- return requiresRecompile;
- }
-
- /**
- *
- * @param oldCallVars
- * @param callVars
- * @param sb
- * @return
- */
- public static boolean reconcileUpdatedCallVarsLoops( RecompileStatus oldCallStatus, RecompileStatus callStatus, StatementBlock sb )
- {
- boolean requiresRecompile = false;
-
- //handle matrices
- for( String varname : sb.variablesUpdated().getVariableNames() )
- {
- MatrixCharacteristics dat1 = oldCallStatus.getTWriteStats().get(varname);
- MatrixCharacteristics dat2 = callStatus.getTWriteStats().get(varname);
- if( dat1!=null && dat2!=null )
- {
- MatrixCharacteristics mcOld = dat1;
- MatrixCharacteristics mc = dat2;
-
- if( mcOld.getRows() != mc.getRows()
- || mcOld.getCols() != mc.getCols()
- || mcOld.getNonZeros() != mc.getNonZeros() )
- {
- long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
- //handle row dimension change in body
- if( mcOld.getRows() != mc.getRows() ) {
- ldim1 = -1;
- requiresRecompile = true;
- }
- //handle column dimension change in body
- if( mcOld.getCols() != mc.getCols() ) {
- ldim2 = -1;
- requiresRecompile = true;
- }
- //handle sparsity change
- if( mcOld.getNonZeros() != mc.getNonZeros() ) {
- lnnz = -1;
- requiresRecompile = true;
- }
-
- MatrixCharacteristics moNew = new MatrixCharacteristics(ldim1, ldim2, -1, -1, lnnz);
- callStatus.getTWriteStats().put(varname, moNew);
- }
- }
- }
-
- return requiresRecompile;
- }
-
- /**
- *
- * @param oldCallVars
- * @param callVarsIf
- * @param callVarsElse
- * @param sb
- * @return
- */
- public static LocalVariableMap reconcileUpdatedCallVarsIf( LocalVariableMap oldCallVars, LocalVariableMap callVarsIf, LocalVariableMap callVarsElse, StatementBlock sb )
- {
- for( String varname : sb.variablesUpdated().getVariableNames() )
- {
- Data origVar = oldCallVars.get(varname);
- Data ifVar = callVarsIf.get(varname);
- Data elseVar = callVarsElse.get(varname);
- Data dat1 = null, dat2 = null;
-
- if( ifVar!=null && elseVar!=null ){ // both branches exists
- dat1 = ifVar;
- dat2 = elseVar;
- }
- else if( ifVar!=null && elseVar==null ){ //only if
- dat1 = origVar;
- dat2 = ifVar;
- }
- else { //only else
- dat1 = origVar;
- dat2 = elseVar;
- }
-
- //compare size and value information (note: by definition both dat1 and dat2 are of same type
- //because we do not allow data type changes)
- if( dat1 != null && dat1 instanceof MatrixObject && dat2!=null )
- {
- //handle matrices
- if( dat1 instanceof MatrixObject && dat2 instanceof MatrixObject )
- {
- MatrixObject moOld = (MatrixObject) dat1;
- MatrixObject mo = (MatrixObject) dat2;
- MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
- MatrixCharacteristics mc = mo.getMatrixCharacteristics();
-
- if( mcOld.getRows() != mc.getRows()
- || mcOld.getCols() != mc.getCols()
- || mcOld.getNonZeros() != mc.getNonZeros() )
- {
- long ldim1 =mc.getRows(), ldim2=mc.getCols(), lnnz=mc.getNonZeros();
-
- //handle row dimension change
- if( mcOld.getRows() != mc.getRows() ) {
- ldim1 = -1; //unknown
- }
- if( mcOld.getCols() != mc.getCols() ) {
- ldim2 = -1; //unknown
- }
- //handle sparsity change
- if( mcOld.getNonZeros() != mc.getNonZeros() ) {
- lnnz = -1; //unknown
- }
-
- MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
- callVarsIf.put(varname, moNew);
- }
- }
- }
- }
-
- return callVarsIf;
- }
-
- /**
- *
- * @param oldStatus
- * @param callStatusIf
- * @param callStatusElse
- * @param sb
- * @return
- */
- public static RecompileStatus reconcileUpdatedCallVarsIf( RecompileStatus oldStatus, RecompileStatus callStatusIf, RecompileStatus callStatusElse, StatementBlock sb )
- {
- for( String varname : sb.variablesUpdated().getVariableNames() )
- {
- MatrixCharacteristics origVar = oldStatus.getTWriteStats().get(varname);
- MatrixCharacteristics ifVar = callStatusIf.getTWriteStats().get(varname);
- MatrixCharacteristics elseVar = callStatusElse.getTWriteStats().get(varname);
- MatrixCharacteristics dat1 = null, dat2 = null;
-
- if( ifVar!=null && elseVar!=null ){ // both branches exists
- dat1 = ifVar;
- dat2 = elseVar;
- }
- else if( ifVar!=null && elseVar==null ){ //only if
- dat1 = origVar;
- dat2 = ifVar;
- }
- else { //only else
- dat1 = origVar;
- dat2 = elseVar;
- }
-
- //compare size and value information (note: by definition both dat1 and dat2 are of same type
- //because we do not allow data type changes)
- if( dat1 != null && dat2!=null )
- {
- MatrixCharacteristics mcOld = dat1;
- MatrixCharacteristics mc = dat2;
-
- if( mcOld.getRows() != mc.getRows()
- || mcOld.getCols() != mc.getCols()
- || mcOld.getNonZeros() != mc.getNonZeros() )
- {
- long ldim1 = (mcOld.getRows()>=0 && mc.getRows()>=0) ?
- Math.max( mcOld.getRows(), mc.getRows() ) : -1;
- long ldim2 = (mcOld.getCols()>=0 && mc.getCols()>=0) ?
- Math.max( mcOld.getCols(), mc.getCols() ) : -1;
- long lnnz = (mcOld.getNonZeros()>=0 && mc.getNonZeros()>=0) ?
- Math.max( mcOld.getNonZeros(), mc.getNonZeros() ) : -1;
-
- MatrixCharacteristics mcNew = new MatrixCharacteristics(ldim1, ldim2, -1, -1, lnnz);
- callStatusIf.getTWriteStats().put(varname, mcNew);
- }
- }
- }
-
- return callStatusIf;
- }
-
- /**
- *
- * @param hops
- * @return
- */
- private static boolean containsRootFunctionOp( ArrayList<Hop> hops )
- {
- boolean ret = false;
- for( Hop h : hops )
- if( h instanceof FunctionOp )
- ret |= true;
-
- return ret;
- }
-
- /**
- *
- * @param dim1
- * @param dim2
- * @param nnz
- * @return
- */
- private static MatrixObject createOutputMatrix( long dim1, long dim2, long nnz )
- {
- MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
- MatrixCharacteristics mc = new MatrixCharacteristics(
- dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
- nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
- moOut.setMetaData(meta);
-
- return moOut;
- }
-
-
- //helper functions for predicate recompile
-
- /**
- *
- * @param ipb
- * @param isb
- * @param vars
- * @param tid
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- private static void recompileIfPredicate( IfProgramBlock ipb, IfStatementBlock isb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- if( isb != null )
- {
- Hop hops = isb.getPredicateHops();
- if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(hops, vars, status, true, tid);
- ipb.setPredicate( tmp );
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && resetRecompile )
- {
- Hop.resetRecompilationFlag(hops, ExecType.CP);
- isb.updatePredicateRecompilationFlag();
- }
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- if( hops instanceof LiteralOp )
- ipb.setPredicateResultVar(((LiteralOp)hops).getName().toLowerCase());
- }
- }
- }
-
- /**
- *
- * @param wpb
- * @param wsb
- * @param vars
- * @param tid
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- private static void recompileWhilePredicate( WhileProgramBlock wpb, WhileStatementBlock wsb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- if( wsb != null )
- {
- Hop hops = wsb.getPredicateHops();
- if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(hops, vars, status, true, tid);
- wpb.setPredicate( tmp );
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && resetRecompile )
- {
- Hop.resetRecompilationFlag(hops, ExecType.CP);
- wsb.updatePredicateRecompilationFlag();
- }
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- if( hops instanceof LiteralOp )
- wpb.setPredicateResultVar(((LiteralOp)hops).getName().toLowerCase());
- }
- }
- }
-
- /**
- *
- * @param fpb
- * @param fsb
- * @param vars
- * @param tid
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- private static void recompileForPredicates( ForProgramBlock fpb, ForStatementBlock fsb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
- throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException
- {
- if( fsb != null )
- {
- Hop fromHops = fsb.getFromHops();
- Hop toHops = fsb.getToHops();
- Hop incrHops = fsb.getIncrementHops();
-
- //handle recompilation flags
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && resetRecompile )
- {
- if( fromHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, tid);
- fpb.setFromInstructions(tmp);
- Hop.resetRecompilationFlag(fromHops,ExecType.CP);
- }
- if( toHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, tid);
- fpb.setToInstructions(tmp);
- Hop.resetRecompilationFlag(toHops,ExecType.CP);
- }
- if( incrHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, tid);
- fpb.setIncrementInstructions(tmp);
- Hop.resetRecompilationFlag(incrHops,ExecType.CP);
- }
- fsb.updatePredicateRecompilationFlags();
- }
- else //no reset of recompilation flags
- {
- if( fromHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, tid);
- fpb.setFromInstructions(tmp);
- }
- if( toHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, tid);
- fpb.setToInstructions(tmp);
- }
- if( incrHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, tid);
- fpb.setIncrementInstructions(tmp);
- }
- }
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- String[] itervars = fpb.getIterablePredicateVars();
- if( fromHops != null && fromHops instanceof LiteralOp )
- itervars[1] = ((LiteralOp)fromHops).getName();
- if( toHops != null && toHops instanceof LiteralOp )
- itervars[2] = ((LiteralOp)toHops).getName();
- if( incrHops != null && incrHops instanceof LiteralOp )
- itervars[3] = ((LiteralOp)incrHops).getName();
- }
- }
-
- /**
- *
- * @param pb
- * @param tid
- * @throws HopsException
- * @throws DMLRuntimeException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- private static void rRecompileProgramBlock2Forced( ProgramBlock pb, long tid, HashSet<String> fnStack, ExecType et )
- throws HopsException, DMLRuntimeException, LopsException, DMLUnsupportedOperationException, IOException
- {
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock pbTmp = (WhileProgramBlock)pb;
- WhileStatementBlock sbTmp = (WhileStatementBlock)pbTmp.getStatementBlock();
- //recompile predicate
- if( sbTmp!=null && !(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)) )
- pbTmp.setPredicate( Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et) );
-
- //recompile body
- for (ProgramBlock pb2 : pbTmp.getChildBlocks())
- rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock pbTmp = (IfProgramBlock)pb;
- IfStatementBlock sbTmp = (IfStatementBlock)pbTmp.getStatementBlock();
- //recompile predicate
- if( sbTmp!=null &&!(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)) )
- pbTmp.setPredicate( Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et) );
- //recompile body
- for( ProgramBlock pb2 : pbTmp.getChildBlocksIfBody() )
- rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
- for( ProgramBlock pb2 : pbTmp.getChildBlocksElseBody() )
- rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock pbTmp = (ForProgramBlock)pb;
- ForStatementBlock sbTmp = (ForStatementBlock) pbTmp.getStatementBlock();
- //recompile predicate
- if( sbTmp!=null &&!(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getFromInstructions(), true, true)) )
- pbTmp.setFromInstructions( Recompiler.recompileHopsDag2Forced(sbTmp.getFromHops(), tid, et) );
- if( sbTmp!=null &&!(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getToInstructions(), true, true)) )
- pbTmp.setToInstructions( Recompiler.recompileHopsDag2Forced(sbTmp.getToHops(), tid, et) );
- if( sbTmp!=null &&!(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getIncrementInstructions(), true, true)) )
- pbTmp.setIncrementInstructions( Recompiler.recompileHopsDag2Forced(sbTmp.getIncrementHops(), tid, et) );
- //recompile body
- for( ProgramBlock pb2 : pbTmp.getChildBlocks() )
- rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
- }
- else if ( pb instanceof FunctionProgramBlock )//includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
- {
- FunctionProgramBlock tmp = (FunctionProgramBlock)pb;
- for( ProgramBlock pb2 : tmp.getChildBlocks() )
- rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
- }
- else
- {
- StatementBlock sb = pb.getStatementBlock();
-
- //recompile hops dag to CP (opt: don't recompile if CP and no MR inst)
- if( sb != null && !(et==ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pb, true, true)) )
- {
- ArrayList<Instruction> tmp = pb.getInstructions();
- tmp = Recompiler.recompileHopsDag2Forced(sb, sb.get_hops(), tid, et);
- pb.setInstructions( tmp );
- }
-
- //recompile functions
- if( OptTreeConverter.containsFunctionCallInstruction(pb) )
- {
- ArrayList<Instruction> tmp = pb.getInstructions();
- for( Instruction inst : tmp )
- if( inst instanceof FunctionCallCPInstruction )
- {
- FunctionCallCPInstruction func = (FunctionCallCPInstruction)inst;
- String fname = func.getFunctionName();
- String fnamespace = func.getNamespace();
- String fKey = DMLProgram.constructFunctionKey(fnamespace, fname);
-
- if( !fnStack.contains(fKey) ) //memoization for multiple calls, recursion
- {
- fnStack.add(fKey);
-
- FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fnamespace, fname);
- rRecompileProgramBlock2Forced(fpb, tid, fnStack, et); //recompile chains of functions
- }
- }
- }
- }
-
- }
-
- /**
- *
- * @param callVars
- * @param sb
- */
- public static void removeUpdatedScalars( LocalVariableMap callVars, StatementBlock sb )
- {
- if( sb != null )
- {
- //remove update scalar variables from constants
- for( String varname : sb.variablesUpdated().getVariables().keySet() )
- {
- Data dat = callVars.get(varname);
- if( dat != null && dat.getDataType() == DataType.SCALAR )
- {
- callVars.remove(varname);
- }
- }
- }
- }
-
- /**
- *
- * @param hops
- * @param vars
- */
- public static void extractDAGOutputStatistics(ArrayList<Hop> hops, LocalVariableMap vars)
- {
- extractDAGOutputStatistics(hops, vars, true);
- }
-
- /**
- *
- * @param hops
- * @param vars
- */
- public static void extractDAGOutputStatistics(ArrayList<Hop> hops, LocalVariableMap vars, boolean overwrite)
- {
- for( Hop hop : hops ) //for all hop roots
- extractDAGOutputStatistics(hop, vars, overwrite);
- }
-
- /**
- *
- * @param hop
- * @param vars
- */
- public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars)
- {
- extractDAGOutputStatistics(hop, vars, true);
- }
-
- /**
- *
- * @param hop
- * @param vars
- * @param overwrite
- */
- public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite)
- {
- if( hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTWRITE ) //for all writes to symbol table
- //&& hop.getDim1()>0 && hop.getDim2()>0 ) //matrix with known dims
- {
- String varName = hop.getName();
- if( !vars.keySet().contains(varName) || overwrite ) //not existing so far
- {
- //extract matrix sizes for size propagation
- if( hop.getDataType()==DataType.MATRIX )
- {
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
- MatrixCharacteristics mc = new MatrixCharacteristics(
- hop.getDim1(), hop.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
- hop.getNnz());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
- mo.setMetaData(meta);
- vars.put(varName, mo);
- }
- //extract scalar constants for second constant propagation
- else if( hop.getDataType()==DataType.SCALAR )
- {
- //extract literal assignments
- if( hop.getInput().size()==1 && hop.getInput().get(0) instanceof LiteralOp )
- {
- ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp)hop.getInput().get(0));
- if( constant!=null )
- vars.put(varName, constant);
- }
- //extract constant variable assignments
- else if( hop.getInput().size()==1 && hop.getInput().get(0) instanceof DataOp)
- {
- DataOp dop = (DataOp) hop.getInput().get(0);
- String dopvarname = dop.getName();
- if( dop.isRead() && vars.keySet().contains(dopvarname) )
- {
- ScalarObject constant = (ScalarObject) vars.get(dopvarname);
- vars.put(varName, constant); //no clone because constant
- }
- }
- //extract ncol/nrow variable assignments
- else if( hop.getInput().size()==1 && hop.getInput().get(0) instanceof UnaryOp
- && (((UnaryOp)hop.getInput().get(0)).getOp()==OpOp1.NROW ||
- ((UnaryOp)hop.getInput().get(0)).getOp()==OpOp1.NCOL) )
- {
- UnaryOp uop = (UnaryOp) hop.getInput().get(0);
- if( uop.getOp()==OpOp1.NROW && uop.getInput().get(0).getDim1()>0 )
- vars.put(varName, new IntObject(uop.getInput().get(0).getDim1()));
- else if( uop.getOp()==OpOp1.NCOL && uop.getInput().get(0).getDim2()>0 )
- vars.put(varName, new IntObject(uop.getInput().get(0).getDim2()));
- }
- //remove other updated scalars
- else
- {
- //we need to remove other updated scalars in order to ensure result
- //correctness of recompilation w/o being too conservative
- vars.remove(varName);
- }
- }
- }
- else //already existing: take largest
- {
- Data dat = vars.get(varName);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject)dat;
- MatrixCharacteristics mc = mo.getMatrixCharacteristics();
- if( OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros()>=0)?((double)mc.getNonZeros())/mc.getRows()/mc.getCols():1.0)
- < OptimizerUtils.estimateSize(hop.getDim1(), hop.getDim2()) )
- {
- //update statistics if necessary
- mc.setDimension(hop.getDim1(), hop.getDim2());
- mc.setNonZeros(hop.getNnz());
- }
- }
- else //scalar (just overwrite)
- {
- if( hop.getInput().size()==1 && hop.getInput().get(0) instanceof LiteralOp )
- {
- ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp)hop.getInput().get(0));
- if( constant!=null )
- vars.put(varName, constant);
- }
- }
-
- }
- }
- }
-
- /**
- * NOTE: no need for update visit status due to early abort
- *
- * @param hop
- * @return
- */
- private static boolean rRequiresRecompile( Hop hop )
- {
- boolean ret = hop.requiresRecompile();
- if( hop.getVisited() == VisitStatus.DONE )
- return ret;
-
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- {
- ret |= rRequiresRecompile(c);
- if( ret ) break; // early abort
- }
-
- hop.setVisited(VisitStatus.DONE);
-
- return ret;
- }
-
- /**
- * Clearing lops for a given hops includes to (1) remove the reference
- * to constructed lops and (2) clear the exec type (for consistency).
- *
- * The latter is important for advanced optimizers like parfor; otherwise subtle
- * side-effects of program recompilation and hop-lop rewrites possible
- * (e.g., see indexingop hop-lop rewrite in combination parfor rewrite set
- * exec type that eventuelly might lead to unnecessary remote_parfor jobs).
- *
- * @param hop
- */
- public static void rClearLops( Hop hop )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //clear all relevant lops to allow for recompilation
- if( hop instanceof LiteralOp )
- {
- //for literal ops, we just clear parents because always constant
- if( hop.getLops() != null )
- hop.getLops().getOutputs().clear();
- }
- else //GENERAL CASE
- {
- hop.resetExecType(); //remove exec type
- hop.setLops(null); //clear lops
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- rClearLops(c);
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- *
- * @param hop
- * @param vars
- * @throws DMLRuntimeException
- */
- public static void rUpdateStatistics( Hop hop, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //recursively process children
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- rUpdateStatistics(c, vars);
-
- boolean updatedSizeExpr = false;
-
- //update statistics for transient reads according to current statistics
- //(with awareness not to override persistent reads to an existing name)
- if( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType() != DataOpTypes.PERSISTENTREAD )
- {
- DataOp d = (DataOp) hop;
- String varName = d.getName();
- if( vars.keySet().contains( varName ) )
- {
- Data dat = vars.get(varName);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject) dat;
- d.setDim1(mo.getNumRows());
- d.setDim2(mo.getNumColumns());
- d.setNnz(mo.getNnz());
- }
- }
- }
- //special case for persistent reads with unknown size (read-after-write)
- else if( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType() == DataOpTypes.PERSISTENTREAD
- && !hop.dimsKnown() && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.CSV )
- {
- //update hop with read meta data
- DataOp dop = (DataOp) hop;
- tryReadMetaDataFileMatrixCharacteristics(dop);
- }
- //update size expression for rand/seq according to symbol table entries
- else if ( hop instanceof DataGenOp )
- {
- DataGenOp d = (DataGenOp) hop;
- HashMap<String,Integer> params = d.getParamIndexMap();
- if ( d.getOp() == DataGenMethod.RAND || d.getOp()==DataGenMethod.SINIT
- || d.getOp() == DataGenMethod.SAMPLE )
- {
- boolean initUnknown = !d.dimsKnown();
- int ix1 = params.get(DataExpression.RAND_ROWS);
- int ix2 = params.get(DataExpression.RAND_COLS);
- //update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
- d.refreshRowsParameterInformation(d.getInput().get(ix1), vars);
- d.refreshColsParameterInformation(d.getInput().get(ix2), vars);
- updatedSizeExpr = initUnknown & d.dimsKnown();
- }
- else if ( d.getOp() == DataGenMethod.SEQ )
- {
- boolean initUnknown = !d.dimsKnown();
- int ix1 = params.get(Statement.SEQ_FROM);
- int ix2 = params.get(Statement.SEQ_TO);
- int ix3 = params.get(Statement.SEQ_INCR);
- double from = d.computeBoundsInformation(d.getInput().get(ix1), vars);
- double to = d.computeBoundsInformation(d.getInput().get(ix2), vars);
- double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars);
-
- //special case increment
- Hop input3 = d.getInput().get(ix3);
- if ( input3 instanceof BinaryOp && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR
- && from!=Double.MAX_VALUE && to!=Double.MAX_VALUE )
- {
- incr =( from >= to )? -1.0 : 1.0;
- }
-
- if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE && incr!=Double.MAX_VALUE ) {
- d.setDim1( 1 + (long)Math.floor((to-from)/incr) );
- d.setDim2( 1 );
- d.setIncrementValue( incr );
- }
- updatedSizeExpr = initUnknown & d.dimsKnown();
- }
- else {
- throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
- }
- }
- //update size expression for reshape according to symbol table entries
- else if ( hop instanceof ReorgOp
- && ((ReorgOp)(hop)).getOp()==Hop.ReOrgOp.RESHAPE )
- {
- ReorgOp d = (ReorgOp) hop;
- boolean initUnknown = !d.dimsKnown();
- d.refreshRowsParameterInformation(d.getInput().get(1), vars);
- d.refreshColsParameterInformation(d.getInput().get(2), vars);
- updatedSizeExpr = initUnknown & d.dimsKnown();
- }
- //update size expression for indexing according to symbol table entries
- else if( hop instanceof IndexingOp )
- {
- IndexingOp iop = (IndexingOp)hop;
- Hop input2 = iop.getInput().get(1); //inpRowL
- Hop input3 = iop.getInput().get(2); //inpRowU
- Hop input4 = iop.getInput().get(3); //inpColL
- Hop input5 = iop.getInput().get(4); //inpColU
- boolean initUnknown = !iop.dimsKnown();
- double rl = iop.computeBoundsInformation(input2, vars);
- double ru = iop.computeBoundsInformation(input3, vars);
- double cl = iop.computeBoundsInformation(input4, vars);
- double cu = iop.computeBoundsInformation(input5, vars);
- if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE )
- iop.setDim1( (long)(ru-rl+1) );
- if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE )
- iop.setDim2( (long)(cu-cl+1) );
- updatedSizeExpr = initUnknown & iop.dimsKnown();
- }
-
- //propagate statistics along inner nodes of DAG,
- //without overwriting inferred size expressions
- if( !updatedSizeExpr ) {
- hop.refreshSizeInformation();
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- * public interface to package local literal replacement
- *
- * @param hop
- * @param vars
- * @throws DMLRuntimeException
- */
- public static void rReplaceLiterals( Hop hop, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- //public interface
- LiteralReplacement.rReplaceLiterals(hop, vars);
- }
-
- /**
- *
- * @param hop
- * @param pid
- */
- public static void rSetExecType( Hop hop, ExecType etype )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //update function names
- hop.setForcedExecType(etype);
-
- if( hop.getInput() != null )
- for( Hop c : hop.getInput() )
- rSetExecType(c, etype);
-
- hop.setVisited(VisitStatus.DONE);
- }
-
-
- /**
- * Returns true iff (1) all instruction are reblock instructions and (2) all
- * individual reblock operations fit in the current memory budget.
- *
- * @param inst
- * @param pb
- * @return
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public static boolean checkCPReblock(MRJobInstruction inst, MatrixObject[] inputs)
- throws DMLRuntimeException, IOException
- {
- boolean ret = true;
-
- boolean localMode = InfrastructureAnalyzer.isLocalMode();
-
- //check only shuffle inst
- String rdInst = inst.getIv_randInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
- if( (rdInst != null && rdInst.length()>0)
- || (rrInst != null && rrInst.length()>0)
- || (mapInst != null && mapInst.length()>0)
- || (aggInst != null && aggInst.length()>0)
- || (otherInst != null && otherInst.length()>0) )
- {
- ret = false;
- }
-
- //check only reblock inst
- if( ret ) {
- String shuffleInst = inst.getIv_shuffleInstructions();
- String[] instParts = shuffleInst.split( Lop.INSTRUCTION_DELIMITOR );
- for( String rblk : instParts )
- if( !InstructionUtils.getOpCode(rblk).equals(ReBlock.OPCODE)
- && !InstructionUtils.getOpCode(rblk).equals(CSVReBlock.OPCODE) )
- {
- ret = false;
- break;
- }
- }
-
- //check output empty blocks (for outputEmptyBlocks=false, a CP reblock can be
- //counter-productive because any export from CP would reintroduce the empty blocks)
- if( ret ){
- String shuffleInst = inst.getIv_shuffleInstructions();
- String[] instParts = shuffleInst.split( Lop.INSTRUCTION_DELIMITOR );
- for( String rblk : instParts )
- if( InstructionUtils.getOpCode(rblk).equals(ReBlock.OPCODE)
- && rblk.endsWith("false") ) //no output of empty blocks
- {
- ret = false;
- break;
- }
- }
-
- //check recompile memory budget
- if( ret ) {
- for( MatrixObject mo : inputs )
- {
- long rows = mo.getNumRows();
- long cols = mo.getNumColumns();
-
- // If the dimensions are unknown then reblock can not be recompiled into CP
- // Note: unknown dimensions at this point can only happen for CSV files.
- // however, we do a conservative check with the CSV filesize
- if ( rows == -1 || cols == -1 )
- {
- Path path = new Path(mo.getFileName());
- long size = MapReduceTool.getFilesizeOnHDFS(path);
- if( size > CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE || CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE > OptimizerUtils.getLocalMemBudget() )
- {
- ret = false;
- break;
- }
- }
- //default case (known dimensions)
- else
- {
- long nnz = mo.getNnz();
- double sp = OptimizerUtils.getSparsity(rows, cols, nnz);
- double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sp);
- if( !OptimizerUtils.isValidCPDimensions(rows, cols)
- || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sp)
- || mem >= OptimizerUtils.getLocalMemBudget() )
- {
- ret = false;
- break;
- }
- }
- }
- }
-
- //check in-memory reblock size threshold (prevent long single-threaded text read)
- //NOTE: this does not apply to local mode because there text read single-threaded as well
- if( ret && !localMode ) {
- for( MatrixObject mo : inputs )
- {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) mo.getMetaData();
- if(( iimd.getInputInfo()==InputInfo.TextCellInputInfo
- || iimd.getInputInfo()==InputInfo.MatrixMarketInputInfo
- || iimd.getInputInfo()==InputInfo.CSVInputInfo
- || iimd.getInputInfo()==InputInfo.BinaryCellInputInfo)
- && !mo.isDirty() )
- {
- //get file size on hdfs (as indicator for estimated read time)
- Path path = new Path(mo.getFileName());
- long fileSize = MapReduceTool.getFilesizeOnHDFS(path);
- //compute cp reblock size threshold based on available parallelism
- long cpThreshold = CP_REBLOCK_THRESHOLD_SIZE *
- OptimizerUtils.getParallelTextReadParallelism();
-
- if( fileSize > cpThreshold ) {
- ret = false;
- break;
- }
- }
- }
- }
-
- return ret;
- }
-
- /**
- * CP Reblock check for spark instructions; in contrast to MR, we can not
- * rely on the input file sizes because inputs might be passed via rdds.
- *
- * @param mc
- * @return
- * @throws DMLRuntimeException
- */
- public static boolean checkCPReblock(ExecutionContext ec, String varin)
- throws DMLRuntimeException
- {
- MatrixObject in = ec.getMatrixObject(varin);
- MatrixCharacteristics mc = in.getMatrixCharacteristics();
-
- long rows = mc.getRows();
- long cols = mc.getCols();
- long nnz = mc.getNonZeros();
-
- //check valid cp reblock recompilation hook
- if( !OptimizerUtils.ALLOW_DYN_RECOMPILATION
- || !OptimizerUtils.isHybridExecutionMode() )
- {
- return false;
- }
-
- //robustness for usage through mlcontext (key/values of input rdds are not
- //serializable for text; also bufferpool rdd read only supported for binaryblock)
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) in.getMetaData();
- if( in.getRDDHandle() != null && iimd.getInputInfo() != InputInfo.BinaryBlockInputInfo ) {
- return false;
- }
-
- //check valid dimensions and memory requirements
- double sp = OptimizerUtils.getSparsity(rows, cols, nnz);
- double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sp);
- if( !OptimizerUtils.isValidCPDimensions(rows, cols)
- || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sp)
- || mem >= OptimizerUtils.getLocalMemBudget() )
- {
- return false;
- }
-
- //check in-memory reblock size threshold (preference: distributed)
- long estFilesize = (long)(3.5 * mem); //conservative estimate
- long cpThreshold = CP_REBLOCK_THRESHOLD_SIZE *
- OptimizerUtils.getParallelTextReadParallelism();
- return (estFilesize < cpThreshold);
- }
-
- /**
- *
- * @param inst
- * @param inputs
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- * @throws IOException
- */
- public static boolean checkCPTransform(MRJobInstruction inst, MatrixObject[] inputs)
- throws DMLRuntimeException, DMLUnsupportedOperationException, IOException
- {
- boolean ret = true;
-
- MatrixObject input = inputs[0]; // there can only be one input in TRANSFORM job
-
- Path path = new Path(input.getFileName());
- long sizeOnHDFS = MapReduceTool.getFilesizeOnHDFS(path);
-
- // dimensions are not checked here, since the worst case dimensions
- // after transformations (with potential dummycoding) are typically unknown.
-
- if( sizeOnHDFS > CP_TRANSFORM_UNKNOWN_THRESHOLD_SIZE
- || sizeOnHDFS*4 > OptimizerUtils.getLocalMemBudget() )
- ret = false;
- LOG.info("checkCPTransform(): size = " + sizeOnHDFS + ", recompile to CP = " + ret);
- return ret;
- }
-
- /**
- *
- * @param inst
- * @param updatedRandInst
- * @return
- * @throws DMLRuntimeException
- */
- public static boolean checkCPDataGen( MRJobInstruction inst, String updatedRandInst )
- throws DMLRuntimeException
- {
- boolean ret = true;
-
- //check only shuffle inst
- String shuffleInst = inst.getIv_shuffleInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
- if( (shuffleInst != null && shuffleInst.length()>0)
- || (rrInst != null && rrInst.length()>0)
- || (mapInst != null && mapInst.length()>0)
- || (aggInst != null && aggInst.length()>0)
- || (otherInst != null && otherInst.length()>0) )
- {
- ret = false;
- }
-
- //check only rand inst
- if( ret ) {
- String[] instParts = updatedRandInst.split( Lop.INSTRUCTION_DELIMITOR );
- for( String lrandStr : instParts ) {
- if( InstructionUtils.getOpCode(lrandStr).equals(DataGen.RAND_OPCODE) )
- {
- //check recompile memory budget
- RandInstruction lrandInst = (RandInstruction) RandInstruction.parseInstruction(lrandStr);
- long rows = lrandInst.getRows();
- long cols = lrandInst.getCols();
- double sparsity = lrandInst.getSparsity();
- double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sparsity);
- if( !OptimizerUtils.isValidCPDimensions(rows, cols)
- || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sparsity)
- || mem >= OptimizerUtils.getLocalMemBudget() )
- {
- ret = false;
- break;
- }
- }
- else if( InstructionUtils.getOpCode(lrandStr).equals(DataGen.SEQ_OPCODE) )
- {
- //check recompile memory budget
- //(don't account for sparsity because always dense)
- SeqInstruction lrandInst = (SeqInstruction) SeqInstruction.parseInstruction(lrandStr);
- long rows = lrandInst.getRows();
- long cols = lrandInst.getCols();
- double mem = MatrixBlock.estimateSizeInMemory(rows, cols, 1.0d);
- if( !OptimizerUtils.isValidCPDimensions(rows, cols)
- || !OptimizerUtils.isValidCPMatrixSize(rows, cols, 1.0d)
- || mem >= OptimizerUtils.getLocalMemBudget() )
- {
- ret = false;
- break;
- }
- }
- else
- {
- ret = false;
- break;
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param in
- * @param out
- * @throws DMLRuntimeException
- */
- public static void executeInMemoryReblock(ExecutionContext ec, String varin, String varout)
- throws DMLRuntimeException
- {
- MatrixObject in = ec.getMatrixObject(varin);
- MatrixObject out = ec.getMatrixObject(varout);
-
- //read text input matrix (through buffer pool, matrix object carries all relevant
- //information including additional arguments for csv reblock)
- MatrixBlock mb = in.acquireRead();
-
- //set output (incl update matrix characteristics)
- out.acquireModify( mb );
- out.release();
- in.release();
- }
-
- /**
- *
- * @param fname
- * @return
- * @throws DMLRuntimeException
- */
- private static void tryReadMetaDataFileMatrixCharacteristics( DataOp dop )
- throws DMLRuntimeException
- {
- try
- {
- //get meta data filename
- String mtdname = DataExpression.getMTDFileName(dop.getFileName());
-
- JobConf job = ConfigurationManager.getCachedJobConf();
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(mtdname);
- if( fs.exists(path) ){
- BufferedReader br = null;
- try
- {
- br = new BufferedReader(new InputStreamReader(fs.open(path)));
- JSONObject mtd = JSONHelper.parse(br);
-
- DataType dt = DataType.valueOf(String.valueOf(mtd.get(DataExpression.DATATYPEPARAM)).toUpperCase());
- dop.setDataType(dt);
- dop.setValueType(ValueType.valueOf(String.valueOf(mtd.get(DataExpression.VALUETYPEPARAM)).toUpperCase()));
- dop.setDim1((dt==DataType.MATRIX)?Long.parseLong(mtd.get(DataExpression.READROWPARAM).toString()):0);
- dop.setDim2((dt==DataType.MATRIX)?Long.parseLong(mtd.get(DataExpression.READCOLPARAM).toString()):0);
- }
- finally {
- if( br != null )
- br.close();
- }
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/HopDagValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopDagValidator.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/HopDagValidator.java
deleted file mode 100644
index 3ea68c2..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopDagValidator.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.utils.Explain;
-
-/**
- * This class allows to check hop dags for validity, e.g., parent-child linking.
- * It purpose is soley for debugging purposes (enabled in ProgramRewriter).
- *
- */
-public class HopDagValidator
-{
- private static final Log LOG = LogFactory.getLog(HopDagValidator.class.getName());
-
- /**
- *
- * @param roots
- * @throws HopsException
- * @throws DMLRuntimeException
- */
- public static void validateHopDag(ArrayList<Hop> roots)
- throws HopsException
- {
- if( roots == null )
- return;
- try
- {
- Hop.resetVisitStatus(roots);
- for( Hop hop : roots )
- rValidateHop(hop);
- }
- catch(HopsException ex)
- {
- try {
- LOG.error( "\n"+Explain.explainHops(roots) );
- }catch(DMLRuntimeException e){}
-
- throw ex;
- }
- }
-
- /**
- *
- * @param root
- * @throws HopsException
- */
- public static void validateHopDag(Hop root)
- throws HopsException
- {
- if( root == null )
- return;
-
- try
- {
- root.resetVisitStatus();
- rValidateHop(root);
- }
- catch(HopsException ex)
- {
- try {
- LOG.error( "\n"+Explain.explain(root) );
- }catch(DMLRuntimeException e){}
-
- throw ex;
- }
- }
-
- /**
- *
- * @param hop
- * @throws HopsException
- */
- private static void rValidateHop( Hop hop )
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- //check parent linking
- for( Hop parent : hop.getParent() )
- if( !parent.getInput().contains(hop) )
- throw new HopsException("Hop id="+hop.getHopID()+" not properly linked to its parent pid="+parent.getHopID()+" "+parent.getClass().getName());
-
- //check child linking
- for( Hop child : hop.getInput() )
- if( !child.getParent().contains(hop) )
- throw new HopsException("Hop id="+hop.getHopID()+" not properly linked to its child cid="+child.getHopID()+" "+child.getClass().getName());
-
- //check empty childs
- if( hop.getInput().isEmpty() )
- if( !(hop instanceof DataOp || hop instanceof LiteralOp) )
- throw new HopsException("Hop id="+hop.getHopID()+" is not a dataop/literal but has no childs.");
-
- //recursively process childs
- for( Hop child : hop.getInput() )
- rValidateHop(child);
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteRule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteRule.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteRule.java
deleted file mode 100644
index 65bb5ad..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/HopRewriteRule.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-
-/**
- * Base class for all hop rewrites in order to enable generic
- * application of all rules.
- *
- */
-public abstract class HopRewriteRule
-{
-
-
- protected static final Log LOG = LogFactory.getLog(HopRewriteRule.class.getName());
-
- /**
- * Handle a generic (last-level) hop DAG with multiple roots.
- *
- * @param roots
- * @param state
- * @return
- * @throws HopsException
- */
- public abstract ArrayList<Hop> rewriteHopDAGs( ArrayList<Hop> roots, ProgramRewriteStatus state )
- throws HopsException;
-
- /**
- * Handle a predicate hop DAG with exactly one root.
- *
- * @param root
- * @param state
- * @return
- * @throws HopsException
- */
- public abstract Hop rewriteHopDAG( Hop root, ProgramRewriteStatus state )
- throws HopsException;
-}
[32/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
deleted file mode 100644
index 5b1111a..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ /dev/null
@@ -1,2506 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.OpOp4;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: Algebraic Simplifications. Simplifies binary expressions
- * in terms of two major purposes: (1) rewrite binary operations
- * to unary operations when possible (in CP this reduces the memory
- * estimate, in MR this allows map-only operations and hence prevents
- * unnecessary shuffle and sort) and (2) remove binary operations that
- * are in itself are unnecessary (e.g., *1 and /1).
- *
- */
-public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
-{
- private static final Log LOG = LogFactory.getLog(RewriteAlgebraicSimplificationDynamic.class.getName());
-
- //valid aggregation operation types for rowOp to Op conversions (not all operations apply)
- private static AggOp[] LOOKUP_VALID_ROW_COL_AGGREGATE = new AggOp[]{AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX, AggOp.MEAN};
-
- //valid aggregation operation types for empty (sparse-safe) operations (not all operations apply)
- //AggOp.MEAN currently not due to missing count/corrections
- private static AggOp[] LOOKUP_VALID_EMPTY_AGGREGATE = new AggOp[]{AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX, AggOp.PROD, AggOp.TRACE};
-
- //valid unary operation types for empty (sparse-safe) operations (not all operations apply)
- private static OpOp1[] LOOKUP_VALID_EMPTY_UNARY = new OpOp1[]{OpOp1.ABS, OpOp1.SIN, OpOp1.TAN, OpOp1.SQRT, OpOp1.ROUND, OpOp1.CUMSUM};
-
- //valid pseudo-sparse-safe binary operators for wdivmm
- private static OpOp2[] LOOKUP_VALID_WDIVMM_BINARY = new OpOp2[]{OpOp2.MULT, OpOp2.DIV};
-
- //valid unary and binary operators for wumm
- private static OpOp1[] LOOKUP_VALID_WUMM_UNARY = new OpOp1[]{OpOp1.ABS, OpOp1.ROUND, OpOp1.CEIL, OpOp1.FLOOR, OpOp1.EXP, OpOp1.LOG, OpOp1.SQRT, OpOp1.SIGMOID, OpOp1.SPROP};
- private static OpOp2[] LOOKUP_VALID_WUMM_BINARY = new OpOp2[]{OpOp2.MULT, OpOp2.POW};
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return roots;
-
- //one pass rewrite-descend (rewrite created pattern)
- for( Hop h : roots )
- rule_AlgebraicSimplification( h, false );
-
- Hop.resetVisitStatus(roots);
-
- //one pass descend-rewrite (for rollup)
- for( Hop h : roots )
- rule_AlgebraicSimplification( h, true );
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return root;
-
- //one pass rewrite-descend (rewrite created pattern)
- rule_AlgebraicSimplification( root, false );
-
- root.resetVisitStatus();
-
- //one pass descend-rewrite (for rollup)
- rule_AlgebraicSimplification( root, true );
-
- return root;
- }
-
-
- /**
- * Note: X/y -> X * 1/y would be useful because * cheaper than / and sparsesafe; however,
- * (1) the results would be not exactly the same (2 rounds instead of 1) and (2) it should
- * come before constant folding while the other simplifications should come after constant
- * folding. Hence, not applied yet.
- *
- * @throws HopsException
- */
- private void rule_AlgebraicSimplification(Hop hop, boolean descendFirst)
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- //recursively process children
- for( int i=0; i<hop.getInput().size(); i++)
- {
- Hop hi = hop.getInput().get(i);
-
- //process childs recursively first (to allow roll-up)
- if( descendFirst )
- rule_AlgebraicSimplification(hi, descendFirst); //see below
-
- //apply actual simplification rewrites (of childs incl checks)
- hi = removeEmptyRightIndexing(hop, hi, i); //e.g., X[,1] -> matrix(0,ru-rl+1,cu-cl+1), if nnz(X)==0
- hi = removeUnnecessaryRightIndexing(hop, hi, i); //e.g., X[,1] -> X, if output == input size
- hi = removeEmptyLeftIndexing(hop, hi, i); //e.g., X[,1]=Y -> matrix(0,nrow(X),ncol(X)), if nnz(X)==0 and nnz(Y)==0
- hi = removeUnnecessaryLeftIndexing(hop, hi, i); //e.g., X[,1]=Y -> Y, if output == input dims
- hi = fuseLeftIndexingChainToAppend(hop, hi, i); //e.g., X[,1]=A; X[,2]=B -> X=cbind(A,B), iff ncol(X)==2 and col1/2 lix
- hi = removeUnnecessaryCumulativeOp(hop, hi, i); //e.g., cumsum(X) -> X, if nrow(X)==1;
- hi = removeUnnecessaryReorgOperation(hop, hi, i); //e.g., matrix(X) -> X, if output == input dims
- hi = removeUnnecessaryOuterProduct(hop, hi, i); //e.g., X*(Y%*%matrix(1,...) -> X*Y, if Y col vector
- hi = fuseDatagenAndReorgOperation(hop, hi, i); //e.g., t(rand(rows=10,cols=1)) -> rand(rows=1,cols=10), if one dim=1
- hi = simplifyColwiseAggregate(hop, hi, i); //e.g., colsums(X) -> sum(X) or X, if col/row vector
- hi = simplifyRowwiseAggregate(hop, hi, i); //e.g., rowsums(X) -> sum(X) or X, if row/col vector
- hi = simplifyColSumsMVMult(hop, hi, i); //e.g., colSums(X*Y) -> t(Y) %*% X, if Y col vector
- hi = simplifyRowSumsMVMult(hop, hi, i); //e.g., rowSums(X*Y) -> X %*% t(Y), if Y row vector
- hi = simplifyEmptyAggregate(hop, hi, i); //e.g., sum(X) -> 0, if nnz(X)==0
- hi = simplifyEmptyUnaryOperation(hop, hi, i); //e.g., round(X) -> matrix(0,nrow(X),ncol(X)), if nnz(X)==0
- hi = simplifyEmptyReorgOperation(hop, hi, i); //e.g., t(X) -> matrix(0, ncol(X), nrow(X))
- hi = simplifyEmptySortOperation(hop, hi, i); //e.g., order(X) -> seq(1, nrow(X)), if nnz(X)==0
- hi = simplifyEmptyMatrixMult(hop, hi, i); //e.g., X%*%Y -> matrix(0,...), if nnz(Y)==0 | X if Y==matrix(1,1,1)
- hi = simplifyIdentityRepMatrixMult(hop, hi, i); //e.g., X%*%y -> X if y matrix(1,1,1);
- hi = simplifyScalarMatrixMult(hop, hi, i); //e.g., X%*%y -> X*as.scalar(y), if y is a 1-1 matrix
- hi = simplifyMatrixMultDiag(hop, hi, i); //e.g., diag(X)%*%Y -> X*Y, if ncol(Y)==1 / -> Y*X if ncol(Y)>1
- hi = simplifyDiagMatrixMult(hop, hi, i); //e.g., diag(X%*%Y)->rowSums(X*t(Y)); if col vector
- hi = simplifySumDiagToTrace(hi); //e.g., sum(diag(X)) -> trace(X); if col vector
- hi = pushdownBinaryOperationOnDiag(hop, hi, i); //e.g., diag(X)*7 -> diag(X*7); if col vector
- hi = simplifyWeightedSquaredLoss(hop, hi, i); //e.g., sum(W * (X - U %*% t(V)) ^ 2) -> wsl(X, U, t(V), W, true),
- hi = simplifyWeightedSigmoidMMChains(hop, hi, i); //e.g., W * sigmoid(Y%*%t(X)) -> wsigmoid(W, Y, t(X), type)
- hi = simplifyWeightedDivMM(hop, hi, i); //e.g., t(U) %*% (X/(U%*%t(V))) -> wdivmm(X, U, t(V), left)
- hi = simplifyWeightedCrossEntropy(hop, hi, i); //e.g., sum(X*log(U%*%t(V))) -> wcemm(X, U, t(V))
- hi = simplifyWeightedUnaryMM(hop, hi, i); //e.g., X*exp(U%*%t(V)) -> wumm(X, U, t(V), exp)
- hi = simplifyDotProductSum(hop, hi, i); //e.g., sum(v^2) -> t(v)%*%v if ncol(v)==1
- hi = fuseSumSquared(hop, hi, i); //e.g., sum(X^2) -> sumSq(X), if ncol(X)>1
- hi = reorderMinusMatrixMult(hop, hi, i); //e.g., (-t(X))%*%y->-(t(X)%*%y), TODO size
- hi = simplifySumMatrixMult(hop, hi, i); //e.g., sum(A%*%B) -> sum(t(colSums(A))*rowSums(B)), if not dot product / wsloss
- hi = simplifyEmptyBinaryOperation(hop, hi, i); //e.g., X*Y -> matrix(0,nrow(X), ncol(X)) / X+Y->X / X-Y -> X
- hi = simplifyScalarMVBinaryOperation(hi); //e.g., X*y -> X*as.scalar(y), if y is a 1-1 matrix
- hi = simplifyNnzComputation(hop, hi, i); //e.g., sum(ppred(X,0,"!=")) -> literal(nnz(X)), if nnz known
-
- //process childs recursively after rewrites (to investigate pattern newly created by rewrites)
- if( !descendFirst )
- rule_AlgebraicSimplification(hi, descendFirst);
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop removeEmptyRightIndexing(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof IndexingOp ) //indexing op
- {
- Hop input = hi.getInput().get(0);
- if( input.getNnz()==0 && //nnz input known and empty
- HopRewriteUtils.isDimsKnown(hi)) //output dims known
- {
- //remove unnecessary right indexing
- HopRewriteUtils.removeChildReference(parent, hi);
-
- Hop hnew = HopRewriteUtils.createDataGenOpByVal( new LiteralOp(hi.getDim1()),
- new LiteralOp(hi.getDim2()), 0);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
- hi = hnew;
-
- LOG.debug("Applied removeEmptyRightIndexing");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop removeUnnecessaryRightIndexing(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof IndexingOp ) //indexing op
- {
- Hop input = hi.getInput().get(0);
- if( HopRewriteUtils.isEqualSize(hi, input) ) //equal dims
- {
- //equal dims of right indexing input and output -> no need for indexing
-
- //remove unnecessary right indexing
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied removeUnnecessaryRightIndexing");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop removeEmptyLeftIndexing(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof LeftIndexingOp ) //left indexing op
- {
- Hop input1 = hi.getInput().get(0); //lhs matrix
- Hop input2 = hi.getInput().get(1); //rhs matrix
-
- if( input1.getNnz()==0 //nnz original known and empty
- && input2.getNnz()==0 ) //nnz input known and empty
- {
- //remove unnecessary right indexing
- HopRewriteUtils.removeChildReference(parent, hi);
- Hop hnew = HopRewriteUtils.createDataGenOp( input1, 0);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
- hi = hnew;
-
- LOG.debug("Applied removeEmptyLeftIndexing");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop removeUnnecessaryLeftIndexing(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof LeftIndexingOp ) //left indexing op
- {
- Hop input = hi.getInput().get(1); //rhs matrix
-
- if( HopRewriteUtils.isEqualSize(hi, input) ) //equal dims
- {
- //equal dims of left indexing input and output -> no need for indexing
-
- //remove unnecessary right indexing
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied removeUnnecessaryLeftIndexing");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop fuseLeftIndexingChainToAppend(Hop parent, Hop hi, int pos)
- {
- boolean applied = false;
-
- //pattern1: X[,1]=A; X[,2]=B -> X=cbind(A,B)
- if( hi instanceof LeftIndexingOp //first lix
- && HopRewriteUtils.isFullColumnIndexing((LeftIndexingOp)hi)
- && hi.getInput().get(0) instanceof LeftIndexingOp //second lix
- && HopRewriteUtils.isFullColumnIndexing((LeftIndexingOp)hi.getInput().get(0))
- && hi.getInput().get(0).getParent().size()==1 //first lix is single consumer
- && hi.getInput().get(0).getInput().get(0).getDim2() == 2 ) //two column matrix
- {
- Hop input2 = hi.getInput().get(1); //rhs matrix
- Hop pred2 = hi.getInput().get(4); //cl=cu
- Hop input1 = hi.getInput().get(0).getInput().get(1); //lhs matrix
- Hop pred1 = hi.getInput().get(0).getInput().get(4); //cl=cu
-
- if( pred1 instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred1)==1
- && pred2 instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred2)==2
- && input1.getDataType()==DataType.MATRIX && input2.getDataType()==DataType.MATRIX )
- {
- //create new cbind operation and rewrite inputs
- HopRewriteUtils.removeChildReference(parent, hi);
- BinaryOp bop = HopRewriteUtils.createBinary(input1, input2, OpOp2.CBIND);
- HopRewriteUtils.addChildReference(parent, bop, pos);
-
- hi = bop;
- applied = true;
- }
- }
-
- //pattern1: X[1,]=A; X[2,]=B -> X=rbind(A,B)
- if( !applied && hi instanceof LeftIndexingOp //first lix
- && HopRewriteUtils.isFullRowIndexing((LeftIndexingOp)hi)
- && hi.getInput().get(0) instanceof LeftIndexingOp //second lix
- && HopRewriteUtils.isFullRowIndexing((LeftIndexingOp)hi.getInput().get(0))
- && hi.getInput().get(0).getParent().size()==1 //first lix is single consumer
- && hi.getInput().get(0).getInput().get(0).getDim1() == 2 ) //two column matrix
- {
- Hop input2 = hi.getInput().get(1); //rhs matrix
- Hop pred2 = hi.getInput().get(2); //rl=ru
- Hop input1 = hi.getInput().get(0).getInput().get(1); //lhs matrix
- Hop pred1 = hi.getInput().get(0).getInput().get(2); //rl=ru
-
- if( pred1 instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred1)==1
- && pred2 instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred2)==2
- && input1.getDataType()==DataType.MATRIX && input2.getDataType()==DataType.MATRIX )
- {
- //create new cbind operation and rewrite inputs
- HopRewriteUtils.removeChildReference(parent, hi);
- BinaryOp bop = HopRewriteUtils.createBinary(input1, input2, OpOp2.RBIND);
- HopRewriteUtils.addChildReference(parent, bop, pos);
-
- hi = bop;
- applied = true;
-
- LOG.debug("Applied fuseLeftIndexingChainToAppend2 (line "+hi.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop removeUnnecessaryCumulativeOp(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof UnaryOp && ((UnaryOp)hi).isCumulativeUnaryOperation() )
- {
- Hop input = hi.getInput().get(0); //input matrix
-
- if( HopRewriteUtils.isDimsKnown(input) //dims input known
- && input.getDim1()==1 ) //1 row
- {
- OpOp1 op = ((UnaryOp)hi).getOp();
-
- //remove unnecessary unary cumsum operator
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied removeUnnecessaryCumulativeOp: "+op);
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop removeUnnecessaryReorgOperation(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp() == ReOrgOp.RESHAPE ) //reshape operation
- {
- Hop input = hi.getInput().get(0);
-
- if( HopRewriteUtils.isEqualSize(hi, input) ) //equal dims
- {
- //equal dims of reshape input and output -> no need for reshape because
- //byrow always refers to both input/output and hence gives the same result
-
- //remove unnecessary right indexing
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied removeUnnecessaryReshape");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop removeUnnecessaryOuterProduct(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof BinaryOp ) //binary cell operation
- {
- Hop right = hi.getInput().get(1);
-
- //check for column replication
- if( right instanceof AggBinaryOp //matrix mult with datagen
- && right.getInput().get(1) instanceof DataGenOp
- && ((DataGenOp)right.getInput().get(1)).hasConstantValue(1d)
- && right.getInput().get(1).getDim1() == 1 //row vector for replication
- && right.getInput().get(0).getDim2() == 1 ) //column vector for mv binary
- {
- //remove unnecessary outer product
- HopRewriteUtils.removeChildReference(hi, right);
- HopRewriteUtils.addChildReference(hi, right.getInput().get(0) );
- hi.refreshSizeInformation();
-
- //cleanup refs to matrix mult if no remaining consumers
- if( right.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( right );
-
- LOG.debug("Applied removeUnnecessaryOuterProduct1 (line "+right.getBeginLine()+")");
- }
- //check for row replication
- else if( right instanceof AggBinaryOp //matrix mult with datagen
- && right.getInput().get(0) instanceof DataGenOp
- && ((DataGenOp)right.getInput().get(0)).hasConstantValue(1d)
- && right.getInput().get(0).getDim2() == 1 //colunm vector for replication
- && right.getInput().get(1).getDim1() == 1 ) //row vector for mv binary
- {
- //remove unnecessary outer product
- HopRewriteUtils.removeChildReference(hi, right);
- HopRewriteUtils.addChildReference(hi, right.getInput().get(1) );
- hi.refreshSizeInformation();
-
- //cleanup refs to matrix mult if no remaining consumers
- if( right.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( right );
-
- LOG.debug("Applied removeUnnecessaryOuterProduct2 (line "+right.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- @SuppressWarnings("unchecked")
- private Hop fuseDatagenAndReorgOperation(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.TRANSPOSE //transpose
- && hi.getInput().get(0) instanceof DataGenOp //datagen
- && hi.getInput().get(0).getParent().size()==1 ) //transpose only consumer
- {
- DataGenOp dop = (DataGenOp)hi.getInput().get(0);
- if( (dop.getOp() == DataGenMethod.RAND || dop.getOp() == DataGenMethod.SINIT)
- && (dop.getDim1()==1 || dop.getDim2()==1 ))
- {
- //relink all parents and dataop (remove transpose)
- HopRewriteUtils.removeAllChildReferences(hi);
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
- for( int i=0; i<parents.size(); i++ ) {
- Hop lparent = parents.get(i);
- int ppos = HopRewriteUtils.getChildReferencePos(lparent, hi);
- HopRewriteUtils.removeChildReferenceByPos(lparent, hi, ppos);
- HopRewriteUtils.addChildReference(lparent, dop, pos);
- }
-
- //flip rows/cols attributes in datagen
- HashMap<String, Integer> rparams = dop.getParamIndexMap();
- int pos1 = rparams.get(DataExpression.RAND_ROWS);
- int pos2 = rparams.get(DataExpression.RAND_COLS);
- rparams.put(DataExpression.RAND_ROWS, pos2);
- rparams.put(DataExpression.RAND_COLS, pos1);
- dop.refreshSizeInformation();
-
- hi = dop;
-
- LOG.debug("Applied fuseDatagenReorgOperation.");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- @SuppressWarnings("unchecked")
- private Hop simplifyColwiseAggregate( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp uhi = (AggUnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( HopRewriteUtils.isValidOp(uhi.getOp(), LOOKUP_VALID_ROW_COL_AGGREGATE) ) {
- if( uhi.getDirection() == Direction.Col )
- {
- if( input.getDim1() == 1 )
- {
- //remove unnecessary col aggregation for 1 row
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied simplifyColwiseAggregate1");
- }
- else if( input.getDim2() == 1 )
- {
- //get old parents (before creating cast over aggregate)
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
-
- //simplify col-aggregate to full aggregate
- uhi.setDirection(Direction.RowCol);
- uhi.setDataType(DataType.SCALAR);
-
- //create cast to keep same output datatype
- UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
-
- //rehang cast under all parents
- for( Hop p : parents ) {
- int ix = HopRewriteUtils.getChildReferencePos(p, hi);
- HopRewriteUtils.removeChildReference(p, hi);
- HopRewriteUtils.addChildReference(p, cast, ix);
- }
-
- hi = cast;
-
- LOG.debug("Applied simplifyColwiseAggregate2");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- @SuppressWarnings("unchecked")
- private Hop simplifyRowwiseAggregate( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp uhi = (AggUnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( HopRewriteUtils.isValidOp(uhi.getOp(), LOOKUP_VALID_ROW_COL_AGGREGATE) ) {
- if( uhi.getDirection() == Direction.Row )
- {
- if( input.getDim2() == 1 )
- {
- //remove unnecessary row aggregation for 1 col
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, input, pos);
- parent.refreshSizeInformation();
- hi = input;
-
- LOG.debug("Applied simplifyRowwiseAggregate1");
- }
- else if( input.getDim1() == 1 )
- {
- //get old parents (before creating cast over aggregate)
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
-
- //simplify row-aggregate to full aggregate
- uhi.setDirection(Direction.RowCol);
- uhi.setDataType(DataType.SCALAR);
-
- //create cast to keep same output datatype
- UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
-
- //rehang cast under all parents
- for( Hop p : parents ) {
- int ix = HopRewriteUtils.getChildReferencePos(p, hi);
- HopRewriteUtils.removeChildReference(p, hi);
- HopRewriteUtils.addChildReference(p, cast, ix);
- }
-
- hi = cast;
-
- LOG.debug("Applied simplifyRowwiseAggregate2");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyColSumsMVMult( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- //colSums(X*Y) -> t(Y) %*% X, if Y col vector; additional transpose later
- //removed by other rewrite if unnecessary, i.e., if Y==t(Z)
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp uhi = (AggUnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( uhi.getOp() == AggOp.SUM && uhi.getDirection() == Direction.Col ) //colsums
- {
- if( input instanceof BinaryOp && ((BinaryOp)input).getOp()==OpOp2.MULT ) //b(*)
- {
- Hop left = input.getInput().get(0);
- Hop right = input.getInput().get(1);
-
- if( left.getDim1()>1 && left.getDim2()>1
- && right.getDim1()>1 && right.getDim2()==1 ) // MV (col vector)
- {
- //remove link parent to rowsums
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create new operators
- ReorgOp trans = HopRewriteUtils.createTranspose(right);
- AggBinaryOp mmult = HopRewriteUtils.createMatrixMultiply(trans, left);
-
- //relink new child
- HopRewriteUtils.addChildReference(parent, mmult, pos);
- hi = mmult;
-
- //cleanup old dag
- if( uhi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(uhi);
- if( input.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(input);
-
- LOG.debug("Applied simplifyColSumsMVMult");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyRowSumsMVMult( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- //rowSums(X * Y) -> X %*% t(Y), if Y row vector; additional transpose later
- //removed by other rewrite if unnecessary, i.e., if Y==t(Z)
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp uhi = (AggUnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( uhi.getOp() == AggOp.SUM && uhi.getDirection() == Direction.Row ) //rowsums
- {
- if( input instanceof BinaryOp && ((BinaryOp)input).getOp()==OpOp2.MULT ) //b(*)
- {
- Hop left = input.getInput().get(0);
- Hop right = input.getInput().get(1);
-
- if( left.getDim1()>1 && left.getDim2()>1
- && right.getDim1()==1 && right.getDim2()>1 ) // MV (row vector)
- {
- //remove link parent to rowsums
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create new operators
- ReorgOp trans = HopRewriteUtils.createTranspose(right);
- AggBinaryOp mmult = HopRewriteUtils.createMatrixMultiply(left, trans);
-
- //relink new child
- HopRewriteUtils.addChildReference(parent, mmult, pos);
- hi = mmult;
-
- //cleanup old dag
- if( uhi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(uhi);
- if( input.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(input);
-
- LOG.debug("Applied simplifyRowSumsMVMult");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptyAggregate(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp uhi = (AggUnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( HopRewriteUtils.isValidOp(uhi.getOp(), LOOKUP_VALID_EMPTY_AGGREGATE) ){
-
- if( HopRewriteUtils.isEmpty(input) )
- {
- //remove unnecessary aggregation
- HopRewriteUtils.removeChildReference(parent, hi);
-
- Hop hnew = null;
- if( uhi.getDirection() == Direction.RowCol )
- hnew = new LiteralOp(0.0);
- else if( uhi.getDirection() == Direction.Col )
- hnew = HopRewriteUtils.createDataGenOp(uhi, input, 0); //nrow(uhi)=1
- else //if( uhi.getDirection() == Direction.Row )
- hnew = HopRewriteUtils.createDataGenOp(input, uhi, 0); //ncol(uhi)=1
-
- //add new child to parent input
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptyAggregate");
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptyUnaryOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof UnaryOp )
- {
- UnaryOp uhi = (UnaryOp)hi;
- Hop input = uhi.getInput().get(0);
-
- if( HopRewriteUtils.isValidOp(uhi.getOp(), LOOKUP_VALID_EMPTY_UNARY) ){
-
- if( HopRewriteUtils.isEmpty(input) )
- {
- //remove unnecessary aggregation
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create literal add it to parent
- Hop hnew = HopRewriteUtils.createDataGenOp(input, 0);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
-
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptyUnaryOperation");
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptyReorgOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof ReorgOp )
- {
- ReorgOp rhi = (ReorgOp)hi;
- Hop input = rhi.getInput().get(0);
-
- if( HopRewriteUtils.isEmpty(input) ) //empty input
- {
- //reorg-operation-specific rewrite
- Hop hnew = null;
- if( rhi.getOp() == ReOrgOp.TRANSPOSE )
- hnew = HopRewriteUtils.createDataGenOp(input, true, input, true, 0);
- else if( rhi.getOp() == ReOrgOp.DIAG ){
- if( HopRewriteUtils.isDimsKnown(input) ){
- if( input.getDim1()==1 ) //diagv2m
- hnew = HopRewriteUtils.createDataGenOp(input, false, input, true, 0);
- else //diagm2v
- hnew = HopRewriteUtils.createDataGenOpByVal(
- HopRewriteUtils.createValueHop(input,true), new LiteralOp(1), 0);
- }
- }
- else if( rhi.getOp() == ReOrgOp.RESHAPE )
- hnew = HopRewriteUtils.createDataGenOpByVal(rhi.getInput().get(1), rhi.getInput().get(2), 0);
-
- //modify dag if one of the above rules applied
- if( hnew != null ){
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptyReorgOperation");
- }
- }
-
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptySortOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //order(X, indexreturn=FALSE) -> matrix(0,nrow(X),1)
- //order(X, indexreturn=TRUE) -> seq(1,nrow(X),1)
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.SORT )
- {
- ReorgOp rhi = (ReorgOp)hi;
- Hop input = rhi.getInput().get(0);
-
- if( HopRewriteUtils.isEmpty(input) ) //empty input
- {
- //reorg-operation-specific rewrite
- Hop hnew = null;
- boolean ixret = false;
-
- if( rhi.getInput().get(3) instanceof LiteralOp ) //index return known
- {
- ixret = HopRewriteUtils.getBooleanValue((LiteralOp)rhi.getInput().get(3));
- if( ixret )
- hnew = HopRewriteUtils.createSeqDataGenOp(input);
- else
- hnew = HopRewriteUtils.createDataGenOp(input, 0);
- }
-
- //modify dag if one of the above rules applied
- if( hnew != null ){
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptySortOperation (indexreturn="+ixret+").");
- }
- }
-
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptyMatrixMult(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply() ) //X%*%Y -> matrix(0, )
- {
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- if( HopRewriteUtils.isEmpty(left) //one input empty
- || HopRewriteUtils.isEmpty(right) )
- {
- //remove unnecessary matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create datagen and add it to parent
- Hop hnew = HopRewriteUtils.createDataGenOp(left, right, 0);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
-
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptyMatrixMult");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyIdentityRepMatrixMult(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply() ) //X%*%Y -> X, if y is matrix(1,1,1)
- {
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- // X %*% y -> X
- if( HopRewriteUtils.isDimsKnown(right) && right.getDim1()==1 && right.getDim2()==1 && //scalar right
- right instanceof DataGenOp && ((DataGenOp)right).hasConstantValue(1.0)) //matrix(1,)
- {
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, left, pos);
- hi = left;
-
- LOG.debug("Applied simplifyIdentiyMatrixMult");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyScalarMatrixMult(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply() ) //X%*%Y
- {
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- // y %*% X -> as.scalar(y) * X
- if( HopRewriteUtils.isDimsKnown(left) && left.getDim1()==1 && left.getDim2()==1 ) //scalar left
- {
- //remove link from parent to matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- UnaryOp cast = new UnaryOp(left.getName(), DataType.SCALAR, ValueType.DOUBLE,
- OpOp1.CAST_AS_SCALAR, left);
- HopRewriteUtils.setOutputParameters(cast, 0, 0, 0, 0, 0);
- BinaryOp mult = new BinaryOp(cast.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cast, right);
- HopRewriteUtils.setOutputParameters(mult, right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), -1);
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
-
- //add mult to parent
- HopRewriteUtils.addChildReference(parent, mult, pos);
- parent.refreshSizeInformation();
-
- hi = mult;
-
- LOG.debug("Applied simplifyScalarMatrixMult1");
- }
- // X %*% y -> X * as.scalar(y)
- else if( HopRewriteUtils.isDimsKnown(right) && right.getDim1()==1 && right.getDim2()==1 ) //scalar right
- {
- //remove link from parent to matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- UnaryOp cast = new UnaryOp(right.getName(), DataType.SCALAR, ValueType.DOUBLE,
- OpOp1.CAST_AS_SCALAR, right);
- HopRewriteUtils.setOutputParameters(cast, 0, 0, 0, 0, 0);
- BinaryOp mult = new BinaryOp(cast.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cast, left);
- HopRewriteUtils.setOutputParameters(mult, left.getDim1(), left.getDim2(), left.getRowsInBlock(), left.getColsInBlock(), -1);
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
-
- //add mult to parent
- HopRewriteUtils.addChildReference(parent, mult, pos);
- parent.refreshSizeInformation();
-
- hi = mult;
-
- LOG.debug("Applied simplifyScalarMatrixMult2");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyMatrixMultDiag(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- Hop hnew = null;
-
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply() ) //X%*%Y
- {
-
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- // diag(X) %*% Y -> X * Y / diag(X) %*% Y -> Y * X
- // previously rep required for the second case: diag(X) %*% Y -> (X%*%ones) * Y
- if( left instanceof ReorgOp && ((ReorgOp)left).getOp()==ReOrgOp.DIAG //left diag
- && HopRewriteUtils.isDimsKnown(left) && left.getDim2()>1 ) //diagV2M
- {
- //System.out.println("diag mm rewrite: dim2(right)="+right.getDim2());
-
- if( right.getDim2()==1 ) //right column vector
- {
- //remove link from parent to matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create binary operation over input and right
- Hop input = left.getInput().get(0); //diag input
- hnew = new BinaryOp(input.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, input, right);
- HopRewriteUtils.setOutputParameters(hnew, left.getDim1(), right.getDim2(), left.getRowsInBlock(), left.getColsInBlock(), -1);
-
- LOG.debug("Applied simplifyMatrixMultDiag1");
- }
- else if( right.getDim2()>1 ) //multi column vector
- {
- //remove link from parent to matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create binary operation over input and right; in contrast to above rewrite,
- //we need to switch the order because MV binary cell operations require vector on the right
- Hop input = left.getInput().get(0); //diag input
- hnew = new BinaryOp(input.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, right, input);
- HopRewriteUtils.setOutputParameters(hnew, left.getDim1(), right.getDim2(), left.getRowsInBlock(), left.getColsInBlock(), -1);
-
- //NOTE: previously to MV binary cell operations we replicated the left (if moderate number of columns: 2)
- //create binary operation over input and right
- //Hop input = left.getInput().get(0);
- //Hop ones = HopRewriteUtils.createDataGenOpByVal(new LiteralOp("1",1), new LiteralOp(String.valueOf(right.getDim2()),right.getDim2()), 1);
- //Hop repmat = new AggBinaryOp( input.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, AggOp.SUM, input, ones );
- //HopRewriteUtils.setOutputParameters(repmat, input.getDim1(), ones.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), -1);
- //hnew = new BinaryOp(input.getName(), DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, repmat, right);
- //HopRewriteUtils.setOutputParameters(hnew, right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), -1);
-
- LOG.debug("Applied simplifyMatrixMultDiag2");
- }
- }
-
- //notes: similar rewrites would be possible for the right side as well, just transposed into the right alignment
- }
-
- //if one of the above rewrites applied
- if( hnew !=null ){
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
-
- //add mult to parent
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
-
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyDiagMatrixMult(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.DIAG && hi.getDim2()==1 ) //diagM2V
- {
- Hop hi2 = hi.getInput().get(0);
- if( hi2 instanceof AggBinaryOp && ((AggBinaryOp)hi2).isMatrixMultiply() ) //X%*%Y
- {
- Hop left = hi2.getInput().get(0);
- Hop right = hi2.getInput().get(1);
-
- //remove link from parent to diag
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //remove links to inputs to matrix mult
- //removeChildReference(hi2, left);
- //removeChildReference(hi2, right);
-
- //create new operators (incl refresh size inside for transpose)
- ReorgOp trans = HopRewriteUtils.createTranspose(right);
- BinaryOp mult = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), OpOp2.MULT, left, trans);
- mult.setRowsInBlock(right.getRowsInBlock());
- mult.setColsInBlock(right.getColsInBlock());
- mult.refreshSizeInformation();
- AggUnaryOp rowSum = new AggUnaryOp(right.getName(), right.getDataType(), right.getValueType(), AggOp.SUM, Direction.Row, mult);
- rowSum.setRowsInBlock(right.getRowsInBlock());
- rowSum.setColsInBlock(right.getColsInBlock());
- rowSum.refreshSizeInformation();
-
- //rehang new subdag under parent node
- HopRewriteUtils.addChildReference(parent, rowSum, pos);
- parent.refreshSizeInformation();
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- hi = rowSum;
-
- LOG.debug("Applied simplifyDiagMatrixMult");
- }
- }
-
- return hi;
- }
-
-
- /**
- *
- * @param hi
- */
- private Hop simplifySumDiagToTrace(Hop hi)
- {
- if( hi instanceof AggUnaryOp )
- {
- AggUnaryOp au = (AggUnaryOp) hi;
- if( au.getOp()==AggOp.SUM && au.getDirection()==Direction.RowCol ) //sum
- {
- Hop hi2 = au.getInput().get(0);
- if( hi2 instanceof ReorgOp && ((ReorgOp)hi2).getOp()==ReOrgOp.DIAG && hi2.getDim2()==1 ) //diagM2V
- {
- Hop hi3 = hi2.getInput().get(0);
-
- //remove diag operator
- HopRewriteUtils.removeChildReference(au, hi2);
- HopRewriteUtils.addChildReference(au, hi3, 0);
-
- //change sum to trace
- au.setOp( AggOp.TRACE );
-
- //cleanup if only consumer of intermediate
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- LOG.debug("Applied simplifySumDiagToTrace");
- }
- }
-
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- @SuppressWarnings("unchecked")
- private Hop pushdownBinaryOperationOnDiag(Hop parent, Hop hi, int pos)
- {
- //diag(X)*7 --> diag(X*7) in order to (1) reduce required memory for b(*) and
- //(2) in order to make the binary operation more efficient (dense vector vs sparse matrix)
- if( hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==OpOp2.MULT )
- {
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- boolean applyLeft = false;
- boolean applyRight = false;
-
- //left input is diag
- if( left instanceof ReorgOp && ((ReorgOp)left).getOp()==ReOrgOp.DIAG
- && left.getParent().size()==1 //binary op only parent
- && left.getInput().get(0).getDim2()==1 //col vector
- && right.getDataType() == DataType.SCALAR )
- {
- applyLeft = true;
- }
- else if( right instanceof ReorgOp && ((ReorgOp)right).getOp()==ReOrgOp.DIAG
- && right.getParent().size()==1 //binary op only parent
- && right.getInput().get(0).getDim2()==1 //col vector
- && left.getDataType() == DataType.SCALAR )
- {
- applyRight = true;
- }
-
- //perform actual rewrite
- if( applyLeft || applyRight )
- {
- //remove all parent links to binary op (since we want to reorder
- //we cannot just look at the current parent)
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
- ArrayList<Integer> parentspos = new ArrayList<Integer>();
- for(Hop lparent : parents) {
- int lpos = HopRewriteUtils.getChildReferencePos(lparent, hi);
- HopRewriteUtils.removeChildReferenceByPos(lparent, hi, lpos);
- parentspos.add(lpos);
- }
-
- //rewire binop-diag-input into diag-binop-input
- if( applyLeft ) {
- Hop input = left.getInput().get(0);
- HopRewriteUtils.removeChildReferenceByPos(hi, left, 0);
- HopRewriteUtils.removeChildReferenceByPos(left, input, 0);
- HopRewriteUtils.addChildReference(left, hi, 0);
- HopRewriteUtils.addChildReference(hi, input, 0);
- hi.refreshSizeInformation();
- hi = left;
- }
- else if ( applyRight ) {
- Hop input = right.getInput().get(0);
- HopRewriteUtils.removeChildReferenceByPos(hi, right, 1);
- HopRewriteUtils.removeChildReferenceByPos(right, input, 0);
- HopRewriteUtils.addChildReference(right, hi, 0);
- HopRewriteUtils.addChildReference(hi, input, 1);
- hi.refreshSizeInformation();
- hi = right;
- }
-
- //relink all parents to the diag operation
- for( int i=0; i<parents.size(); i++ ) {
- Hop lparent = parents.get(i);
- int lpos = parentspos.get(i);
- HopRewriteUtils.addChildReference(lparent, hi, lpos);
- }
-
- LOG.debug("Applied pushdownBinaryOperationOnDiag.");
- }
- }
-
- return hi;
- }
-
-
- /**
- * Searches for weighted squared loss expressions and replaces them with a quaternary operator.
- * Currently, this search includes the following three patterns:
- * 1) sum (W * (X - U %*% t(V)) ^ 2) (post weighting)
- * 2) sum ((X - W * (U %*% t(V))) ^ 2) (pre weighting)
- * 3) sum ((X - (U %*% t(V))) ^ 2) (no weighting)
- *
- * NOTE: We include transpose into the pattern because during runtime we need to compute
- * U%*% t(V) pointwise; having V and not t(V) at hand allows for a cache-friendly implementation
- * without additional memory requirements for internal transpose.
- *
- * This rewrite is conceptually a static rewrite; however, the current MR runtime only supports
- * U/V factors of rank up to the blocksize (1000). We enforce this contraint here during the general
- * rewrite because this is an uncommon case. Also, the intention is to remove this constaint as soon
- * as we generalized the runtime or hop/lop compilation.
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyWeightedSquaredLoss(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //NOTE: there might be also a general simplification without custom operator
- //via (X-UVt)^2 -> X^2 - 2X*UVt + UVt^2
- Hop hnew = null;
-
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getDirection()==Direction.RowCol
- && ((AggUnaryOp)hi).getOp() == AggOp.SUM //all patterns rooted by sum()
- && hi.getInput().get(0) instanceof BinaryOp //all patterns subrooted by binary op
- && hi.getInput().get(0).getDim2() > 1 ) //not applied for vector-vector mult
- {
- BinaryOp bop = (BinaryOp) hi.getInput().get(0);
- boolean appliedPattern = false;
-
- //Pattern 1) sum (W * (X - U %*% t(V)) ^ 2) (post weighting)
- //alternative pattern: sum (W * (U %*% t(V) - X) ^ 2)
- if( bop.getOp()==OpOp2.MULT && bop.getInput().get(1) instanceof BinaryOp
- && bop.getInput().get(0).getDataType()==DataType.MATRIX
- && HopRewriteUtils.isEqualSize(bop.getInput().get(0), bop.getInput().get(1)) //prevent mv
- && ((BinaryOp)bop.getInput().get(1)).getOp()==OpOp2.POW
- && bop.getInput().get(1).getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)bop.getInput().get(1).getInput().get(1))==2)
- {
- Hop W = bop.getInput().get(0);
- Hop tmp = bop.getInput().get(1).getInput().get(0); //(X - U %*% t(V))
-
- if( tmp instanceof BinaryOp && ((BinaryOp)tmp).getOp()==OpOp2.MINUS
- && HopRewriteUtils.isEqualSize(tmp.getInput().get(0), tmp.getInput().get(1)) //prevent mv
- && tmp.getInput().get(0).getDataType() == DataType.MATRIX )
- {
- //a) sum (W * (X - U %*% t(V)) ^ 2)
- int uvIndex = -1;
- if( tmp.getInput().get(1) instanceof AggBinaryOp //ba gurantees matrices
- && HopRewriteUtils.isSingleBlock(tmp.getInput().get(1).getInput().get(0),true)) //BLOCKSIZE CONSTRAINT
- {
- uvIndex = 1;
- }
- //b) sum (W * (U %*% t(V) - X) ^ 2)
- else if(tmp.getInput().get(0) instanceof AggBinaryOp //ba gurantees matrices
- && HopRewriteUtils.isSingleBlock(tmp.getInput().get(0).getInput().get(0),true)) //BLOCKSIZE CONSTRAINT
- {
- uvIndex = 0;
- }
-
- if( uvIndex >= 0 ) //rewrite match
- {
- Hop X = tmp.getInput().get((uvIndex==0)?1:0);
- Hop U = tmp.getInput().get(uvIndex).getInput().get(0);
- Hop V = tmp.getInput().get(uvIndex).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(V) ) {
- V = HopRewriteUtils.createTranspose(V);
- }
- else{
- V = V.getInput().get(0);
- }
-
- //handle special case of post_nz
- if( HopRewriteUtils.isNonZeroIndicator(W, X) ){
- W = new LiteralOp(1);
- }
-
- //construct quaternary hop
- hnew = new QuaternaryOp(hi.getName(), DataType.SCALAR, ValueType.DOUBLE,
- OpOp4.WSLOSS, X, U, V, W, true);
- HopRewriteUtils.setOutputParametersForScalar(hnew);
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSquaredLoss1"+uvIndex+" (line "+hi.getBeginLine()+")");
- }
- }
- }
-
- //Pattern 2) sum ((X - W * (U %*% t(V))) ^ 2) (pre weighting)
- //alternative pattern: sum ((W * (U %*% t(V)) - X) ^ 2)
- if( !appliedPattern
- && bop.getOp()==OpOp2.POW && bop.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)bop.getInput().get(1))==2
- && bop.getInput().get(0) instanceof BinaryOp
- && bop.getInput().get(0).getDataType()==DataType.MATRIX
- && ((BinaryOp)bop.getInput().get(0)).getOp()==OpOp2.MINUS
- && HopRewriteUtils.isEqualSize(bop.getInput().get(0).getInput().get(0), bop.getInput().get(0).getInput().get(1)) //prevent mv
- && bop.getInput().get(0).getInput().get(0).getDataType()==DataType.MATRIX)
- {
- Hop lleft = bop.getInput().get(0).getInput().get(0);
- Hop lright = bop.getInput().get(0).getInput().get(1);
-
- //a) sum ((X - W * (U %*% t(V))) ^ 2)
- int wuvIndex = -1;
- if( lright instanceof BinaryOp && lright.getInput().get(1) instanceof AggBinaryOp ){
- wuvIndex = 1;
- }
- //b) sum ((W * (U %*% t(V)) - X) ^ 2)
- else if( lleft instanceof BinaryOp && lleft.getInput().get(1) instanceof AggBinaryOp ){
- wuvIndex = 0;
- }
-
- if( wuvIndex >= 0 ) //rewrite match
- {
- Hop X = bop.getInput().get(0).getInput().get((wuvIndex==0)?1:0);
- Hop tmp = bop.getInput().get(0).getInput().get(wuvIndex); //(W * (U %*% t(V)))
-
- if( ((BinaryOp)tmp).getOp()==OpOp2.MULT
- && tmp.getInput().get(0).getDataType() == DataType.MATRIX
- && HopRewriteUtils.isEqualSize(tmp.getInput().get(0), tmp.getInput().get(1)) //prevent mv
- && HopRewriteUtils.isSingleBlock(tmp.getInput().get(1).getInput().get(0),true)) //BLOCKSIZE CONSTRAINT
- {
- Hop W = tmp.getInput().get(0);
- Hop U = tmp.getInput().get(1).getInput().get(0);
- Hop V = tmp.getInput().get(1).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(V) ) {
- V = HopRewriteUtils.createTranspose(V);
- }
- else {
- V = V.getInput().get(0);
- }
-
- hnew = new QuaternaryOp(hi.getName(), DataType.SCALAR, ValueType.DOUBLE,
- OpOp4.WSLOSS, X, U, V, W, false);
- HopRewriteUtils.setOutputParametersForScalar(hnew);
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSquaredLoss2"+wuvIndex+" (line "+hi.getBeginLine()+")");
- }
- }
- }
-
- //Pattern 3) sum ((X - (U %*% t(V))) ^ 2) (no weighting)
- //alternative pattern: sum (((U %*% t(V)) - X) ^ 2)
- if( !appliedPattern
- && bop.getOp()==OpOp2.POW && bop.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)bop.getInput().get(1))==2
- && bop.getInput().get(0) instanceof BinaryOp
- && bop.getInput().get(0).getDataType()==DataType.MATRIX
- && ((BinaryOp)bop.getInput().get(0)).getOp()==OpOp2.MINUS
- && HopRewriteUtils.isEqualSize(bop.getInput().get(0).getInput().get(0), bop.getInput().get(0).getInput().get(1)) //prevent mv
- && bop.getInput().get(0).getInput().get(0).getDataType()==DataType.MATRIX)
- {
- Hop lleft = bop.getInput().get(0).getInput().get(0);
- Hop lright = bop.getInput().get(0).getInput().get(1);
-
- //a) sum ((X - (U %*% t(V))) ^ 2)
- int uvIndex = -1;
- if( lright instanceof AggBinaryOp //ba gurantees matrices
- && HopRewriteUtils.isSingleBlock(lright.getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- uvIndex = 1;
- }
- //b) sum (((U %*% t(V)) - X) ^ 2)
- else if( lleft instanceof AggBinaryOp //ba gurantees matrices
- && HopRewriteUtils.isSingleBlock(lleft.getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- uvIndex = 0;
- }
-
- if( uvIndex >= 0 ) //rewrite match
- {
- Hop X = bop.getInput().get(0).getInput().get((uvIndex==0)?1:0);
- Hop tmp = bop.getInput().get(0).getInput().get(uvIndex); //(U %*% t(V))
- Hop W = new LiteralOp(1); //no weighting
- Hop U = tmp.getInput().get(0);
- Hop V = tmp.getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(V) ) {
- V = HopRewriteUtils.createTranspose(V);
- }
- else {
- V = V.getInput().get(0);
- }
-
- hnew = new QuaternaryOp(hi.getName(), DataType.SCALAR, ValueType.DOUBLE,
- OpOp4.WSLOSS, X, U, V, W, false);
- HopRewriteUtils.setOutputParametersForScalar(hnew);
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSquaredLoss3"+uvIndex+" (line "+hi.getBeginLine()+")");
- }
- }
- }
-
- //relink new hop into original position
- if( hnew != null ) {
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyWeightedSigmoidMMChains(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- Hop hnew = null;
-
- if( hi instanceof BinaryOp //all patterns subrooted by W *
- && ((BinaryOp) hi).getOp()==OpOp2.MULT
- && hi.getDim2() > 1 //not applied for vector-vector mult
- && HopRewriteUtils.isEqualSize(hi.getInput().get(0), hi.getInput().get(1)) //prevent mv
- && hi.getInput().get(0).getDataType()==DataType.MATRIX
- && hi.getInput().get(1) instanceof UnaryOp ) //sigmoid/log
- {
- UnaryOp uop = (UnaryOp) hi.getInput().get(1);
- boolean appliedPattern = false;
-
- //Pattern 1) W * sigmoid(Y%*%t(X)) (basic)
- if( uop.getOp() == OpOp1.SIGMOID
- && uop.getInput().get(0) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(uop.getInput().get(0).getInput().get(0),true) )
- {
- Hop W = hi.getInput().get(0);
- Hop Y = uop.getInput().get(0).getInput().get(0);
- Hop tX = uop.getInput().get(0).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(tX) ) {
- tX = HopRewriteUtils.createTranspose(tX);
- }
- else
- tX = tX.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WSIGMOID, W, Y, tX, false, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSigmoid1 (line "+hi.getBeginLine()+")");
- }
-
- //Pattern 2) W * sigmoid(-(Y%*%t(X))) (minus)
- if( !appliedPattern
- && uop.getOp() == OpOp1.SIGMOID
- && uop.getInput().get(0) instanceof BinaryOp
- && ((BinaryOp)uop.getInput().get(0)).getOp()==OpOp2.MINUS
- && uop.getInput().get(0).getInput().get(0) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe(
- (LiteralOp)uop.getInput().get(0).getInput().get(0))==0
- && uop.getInput().get(0).getInput().get(1) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(uop.getInput().get(0).getInput().get(1).getInput().get(0),true))
- {
- Hop W = hi.getInput().get(0);
- Hop Y = uop.getInput().get(0).getInput().get(1).getInput().get(0);
- Hop tX = uop.getInput().get(0).getInput().get(1).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(tX) ) {
- tX = HopRewriteUtils.createTranspose(tX);
- }
- else
- tX = tX.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WSIGMOID, W, Y, tX, false, true);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSigmoid2 (line "+hi.getBeginLine()+")");
- }
-
- //Pattern 3) W * log(sigmoid(Y%*%t(X))) (log)
- if( !appliedPattern
- && uop.getOp() == OpOp1.LOG
- && uop.getInput().get(0) instanceof UnaryOp
- && ((UnaryOp)uop.getInput().get(0)).getOp() == OpOp1.SIGMOID
- && uop.getInput().get(0).getInput().get(0) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(uop.getInput().get(0).getInput().get(0).getInput().get(0),true) )
- {
- Hop W = hi.getInput().get(0);
- Hop Y = uop.getInput().get(0).getInput().get(0).getInput().get(0);
- Hop tX = uop.getInput().get(0).getInput().get(0).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(tX) ) {
- tX = HopRewriteUtils.createTranspose(tX);
- }
- else
- tX = tX.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WSIGMOID, W, Y, tX, true, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSigmoid3 (line "+hi.getBeginLine()+")");
- }
-
- //Pattern 4) W * log(sigmoid(-(Y%*%t(X)))) (log_minus)
- if( !appliedPattern
- && uop.getOp() == OpOp1.LOG
- && uop.getInput().get(0) instanceof UnaryOp
- && ((UnaryOp)uop.getInput().get(0)).getOp() == OpOp1.SIGMOID
- && uop.getInput().get(0).getInput().get(0) instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp) uop.getInput().get(0).getInput().get(0);
-
- if( bop.getOp() == OpOp2.MINUS
- && bop.getInput().get(0) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)bop.getInput().get(0))==0
- && bop.getInput().get(1) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(bop.getInput().get(1).getInput().get(0),true))
- {
- Hop W = hi.getInput().get(0);
- Hop Y = bop.getInput().get(1).getInput().get(0);
- Hop tX = bop.getInput().get(1).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(tX) ) {
- tX = HopRewriteUtils.createTranspose(tX);
- }
- else
- tX = tX.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WSIGMOID, W, Y, tX, true, true);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedSigmoid4 (line "+hi.getBeginLine()+")");
- }
- }
- }
-
- //relink new hop into original position
- if( hnew != null ) {
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyWeightedDivMM(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- Hop hnew = null;
- boolean appliedPattern = false;
-
- //left/right patterns rooted by 'ab - b(div)' or 'ab - b(mult)'
- //note: we do not rewrite t(X)%*%(w*(X%*%v)) where w and v are vectors (see mmchain ops)
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply()
- && (hi.getInput().get(0) instanceof BinaryOp
- && HopRewriteUtils.isValidOp(((BinaryOp)hi.getInput().get(0)).getOp(), LOOKUP_VALID_WDIVMM_BINARY)
- || hi.getInput().get(1) instanceof BinaryOp
- && hi.getDim2() > 1 //not applied for vector-vector mult
- && HopRewriteUtils.isValidOp(((BinaryOp)hi.getInput().get(1)).getOp(), LOOKUP_VALID_WDIVMM_BINARY)) )
- {
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- //Pattern 1) t(U) %*% (W/(U%*%t(V)))
- //alternative pattern: t(U) %*% (W*(U%*%t(V)))
- if( right instanceof BinaryOp && HopRewriteUtils.isValidOp(((BinaryOp)right).getOp(),LOOKUP_VALID_WDIVMM_BINARY)
- && HopRewriteUtils.isEqualSize(right.getInput().get(0), right.getInput().get(1)) //prevent mv
- && right.getInput().get(1) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(right.getInput().get(1).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = right.getInput().get(0);
- Hop U = right.getInput().get(1).getInput().get(0);
- Hop V = right.getInput().get(1).getInput().get(1);
-
- if( HopRewriteUtils.isTransposeOfItself(left, U) )
- {
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- boolean mult = ((BinaryOp)right).getOp() == OpOp2.MULT;
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, W, U, V, 1, mult, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- //add output transpose for efficient target indexing (redundant t() removed by other rewrites)
- hnew = HopRewriteUtils.createTranspose(hnew);
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM1 (line "+hi.getBeginLine()+")");
- }
- }
-
- //Pattern 2) (W/(U%*%t(V))) %*% V
- //alternative pattern: (W*(U%*%t(V))) %*% V
- if( !appliedPattern
- && left instanceof BinaryOp && HopRewriteUtils.isValidOp(((BinaryOp)left).getOp(), LOOKUP_VALID_WDIVMM_BINARY)
- && HopRewriteUtils.isEqualSize(left.getInput().get(0), left.getInput().get(1)) //prevent mv
- && left.getInput().get(1) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(left.getInput().get(1).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = left.getInput().get(0);
- Hop U = left.getInput().get(1).getInput().get(0);
- Hop V = left.getInput().get(1).getInput().get(1);
-
- if( HopRewriteUtils.isTransposeOfItself(right, V) )
- {
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = right;
- else
- V = V.getInput().get(0);
-
- boolean mult = ((BinaryOp)left).getOp() == OpOp2.MULT;
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, W, U, V, 2, mult, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM2 (line "+hi.getBeginLine()+")");
- }
- }
-
- //Pattern 3) t(U) %*% ((X!=0)*(U%*%t(V)-X))
- if( right instanceof BinaryOp && ((BinaryOp)right).getOp()==LOOKUP_VALID_WDIVMM_BINARY[0] //MULT
- && right.getInput().get(1) instanceof BinaryOp && ((BinaryOp)right.getInput().get(1)).getOp()==OpOp2.MINUS
- && right.getInput().get(1).getInput().get(0) instanceof AggBinaryOp
- && right.getInput().get(1).getInput().get(1).getDataType() == DataType.MATRIX
- && HopRewriteUtils.isSingleBlock(right.getInput().get(1).getInput().get(0).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = right.getInput().get(0);
- Hop U = right.getInput().get(1).getInput().get(0).getInput().get(0);
- Hop V = right.getInput().get(1).getInput().get(0).getInput().get(1);
- Hop X = right.getInput().get(1).getInput().get(1);
-
- if( HopRewriteUtils.isNonZeroIndicator(W, X) //W-X constraint
- && HopRewriteUtils.isTransposeOfItself(left, U) ) //t(U)-U constraint
- {
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, X, U, V, 1, true, true);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- //add output transpose for efficient target indexing (redundant t() removed by other rewrites)
- hnew = HopRewriteUtils.createTranspose(hnew);
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM3 (line "+hi.getBeginLine()+")");
- }
- }
-
- //Pattern 4) ((X!=0)*(U%*%t(V)-X)) %*% V
- if( !appliedPattern
- && left instanceof BinaryOp && ((BinaryOp)left).getOp()==LOOKUP_VALID_WDIVMM_BINARY[0] //MULT
- && left.getInput().get(1) instanceof BinaryOp && ((BinaryOp)left.getInput().get(1)).getOp()==OpOp2.MINUS
- && left.getInput().get(1).getInput().get(0) instanceof AggBinaryOp
- && left.getInput().get(1).getInput().get(1).getDataType() == DataType.MATRIX
- && HopRewriteUtils.isSingleBlock(left.getInput().get(1).getInput().get(0).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = left.getInput().get(0);
- Hop U = left.getInput().get(1).getInput().get(0).getInput().get(0);
- Hop V = left.getInput().get(1).getInput().get(0).getInput().get(1);
- Hop X = left.getInput().get(1).getInput().get(1);
-
- if( HopRewriteUtils.isNonZeroIndicator(W, X) //W-X constraint
- && HopRewriteUtils.isTransposeOfItself(right, V) ) //V-t(V) constraint
- {
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = right;
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, X, U, V, 2, true, true);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM4 (line "+hi.getBeginLine()+")");
- }
- }
- }
-
- //Pattern 5) (W*(U%*%t(V)))
- if( !appliedPattern
- && hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==LOOKUP_VALID_WDIVMM_BINARY[0] //MULT
- && HopRewriteUtils.isEqualSize(hi.getInput().get(0), hi.getInput().get(1)) //prevent mv
- && hi.getDim2() > 1 //not applied for vector-vector mult
- && hi.getInput().get(0).getDataType() == DataType.MATRIX
- && hi.getInput().get(0).getDim2() > hi.getInput().get(0).getColsInBlock()
- && hi.getInput().get(1) instanceof AggBinaryOp
- && (((AggBinaryOp) hi.getInput().get(1)).checkMapMultChain() == ChainType.NONE || hi.getInput().get(1).getInput().get(1).getDim2() > 1) //no mmchain
- && HopRewriteUtils.isSingleBlock(hi.getInput().get(1).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = hi.getInput().get(0);
- Hop U = hi.getInput().get(1).getInput().get(0);
- Hop V = hi.getInput().get(1).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, W, U, V, 0, true, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM5 (line "+hi.getBeginLine()+")");
- }
-
- //relink new hop into original position
- if( hnew != null ) {
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyWeightedCrossEntropy(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- Hop hnew = null;
-
- //Pattern 1) sum( X * log(U %*% t(V)))
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getDirection()==Direction.RowCol
- && ((AggUnaryOp)hi).getOp() == AggOp.SUM //pattern rooted by sum()
- && hi.getInput().get(0) instanceof BinaryOp //pattern subrooted by binary op
- && hi.getInput().get(0).getDim2() > 1 ) //not applied for vector-vector mult
- {
- BinaryOp bop = (BinaryOp) hi.getInput().get(0);
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- if( bop.getOp()==OpOp2.MULT && left.getDataType()==DataType.MATRIX
- && HopRewriteUtils.isEqualSize(left, right) //prevent mb
- && right instanceof UnaryOp && ((UnaryOp)right).getOp()==OpOp1.LOG
- && right.getInput().get(0) instanceof AggBinaryOp //ba gurantees matrices
- && HopRewriteUtils.isSingleBlock(right.getInput().get(0).getInput().get(0),true)) //BLOCKSIZE CONSTRAINT
- {
- Hop X = left;
- Hop U = right.getInput().get(0).getInput().get(0);
- Hop V = right.getInput().get(0).getInput().get(1);
-
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.SCALAR, ValueType.DOUBLE, OpOp4.WCEMM, X, U, V);
- HopRewriteUtils.setOutputBlocksizes(hnew, X.getRowsInBlock(), X.getColsInBlock());
-
- LOG.debug("Applied simplifyWeightedCEMM (line "+hi.getBeginLine()+")");
- }
- }
-
- //relink new hop into original position
- if( hnew != null ) {
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyWeightedUnaryMM(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- Hop hnew = null;
- boolean appliedPattern = false;
-
- //Pattern 1) (W*uop(U%*%t(V)))
- if( hi instanceof BinaryOp && HopRewriteUtils.isValidOp(((BinaryOp)hi).getOp(),LOOKUP_VALID_WDIVMM_BINARY)
- && HopRewriteUtils.isEqualSize(hi.getInput().get(0), hi.getInput().get(1)) //prevent mv
- && hi.getDim2() > 1 //not applied for vector-vector mult
- && hi.getInput().get(0).getDataType() == DataType.MATRIX
- && hi.getInput().get(0).getDim2() > hi.getInput().get(0).getColsInBlock()
- && hi.getInput().get(1) instanceof UnaryOp
- && HopRewriteUtils.isValidOp(((UnaryOp)hi.getInput().get(1)).getOp(), LOOKUP_VALID_WUMM_UNARY)
- && hi.getInput().get(1).getInput().get(0) instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(hi.getInput().get(1).getInput().get(0).getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- Hop W = hi.getInput().get(0);
- Hop U = hi.getInput().get(1).getInput().get(0).getInput().get(0);
- Hop V = hi.getInput().get(1).getInput().get(0).getInput().get(1);
- boolean mult = ((BinaryOp)hi).getOp()==OpOp2.MULT;
- OpOp1 op = ((UnaryOp)hi.getInput().get(1)).getOp();
-
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WUMM, W, U, V, mult, op, null);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedUnaryMM1 (line "+hi.getBeginLine()+")");
- }
-
- //Pattern 2) (W*sop(U%*%t(V),c)) for known sop translating to unary ops
- if( !appliedPattern
- && hi instanceof BinaryOp && HopRewriteUtils.isValidOp(((BinaryOp)hi).getOp(),LOOKUP_VALID_WDIVMM_BINARY)
- && HopRewriteUtils.isEqualSize(hi.getInput().get(0), hi.getInput().get(1)) //prevent mv
- && hi.getDim2() > 1 //not applied for vector-vector mult
- && hi.getInput().get(0).getDataType() == DataType.MATRIX
- && hi.getInput().get(0).getDim2() > hi.getInput().get(0).getColsInBlock()
- && hi.getInput().get(1) instanceof BinaryOp
- && HopRewriteUtils.isValidOp(((BinaryOp)hi.getInput().get(1)).getOp(), LOOKUP_VALID_WUMM_BINARY) )
- {
- Hop left = hi.getInput().get(1).getInput().get(0);
- Hop right = hi.getInput().get(1).getInput().get(1);
- Hop abop = null;
-
- //pattern 2a) matrix-scalar operations
- if( right.getDataType()==DataType.SCALAR && right instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)right)==2 //pow2, mult2
- && left instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(left.getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- abop = left;
- }
- //pattern 2b) scalar-matrix operations
- else if( left.getDataType()==DataType.SCALAR && left instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)left)==2 //mult2
- && ((BinaryOp)hi.getInput().get(1)).getOp() == OpOp2.MULT
- && right instanceof AggBinaryOp
- && HopRewriteUtils.isSingleBlock(right.getInput().get(0),true) ) //BLOCKSIZE CONSTRAINT
- {
- abop = right;
- }
-
- if( abop != null ) {
- Hop W = hi.getInput().get(0);
- Hop U = abop.getInput().get(0);
- Hop V = abop.getInput().get(1);
- boolean mult = ((BinaryOp)hi).getOp()==OpOp2.MULT;
- OpOp2 op = ((BinaryOp)hi.getInput().get(1)).getOp();
-
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
-
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WUMM, W, U, V, mult, null, op);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedUnaryMM2 (line "+hi.getBeginLine()+")");
- }
- }
-
-
- //relink new hop into original position
- if( hnew != null ) {
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
- }
-
- return hi;
- }
-
- /**
- * NOTE: dot-product-sum could be also applied to sum(a*b). However, we
- * restrict ourselfs to sum(a^2) and transitively sum(a*a) since a general mm
- * a%*%b on MR can be also counter-productive (e.g., MMCJ) while tsmm is always
- * beneficial.
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyDotProductSum(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //sum(v^2)/sum(v1*v2) --> as.scalar(t(v)%*%v) in order to exploit tsmm vector dotproduct
- //w/o materialization of intermediates
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getOp()==AggOp.SUM //sum
- && ((AggUnaryOp)hi).getDirection()==Direction.RowCol //full aggregate
- && hi.getInput().get(0).getDim2() == 1 ) //vector (for correctness)
- {
- Hop baLeft = null;
- Hop baRight = null;
-
- Hop hi2 = hi.getInput().get(0); //check for ^2 w/o multiple consumers
- //check for sum(v^2), might have been rewritten from sum(v*v)
- if( hi2 instanceof BinaryOp && ((BinaryOp)hi2).getOp()==OpOp2.POW
- && hi2.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)hi2.getInput().get(1))==2
- && hi2.getParent().size() == 1 ) //no other consumer than sum
- {
- Hop input = hi2.getInput().get(0);
- baLeft = input;
- baRight = input;
- }
- //check for sum(v1*v2), but prevent to rewrite sum(v1*v2*v3) which is later compiled into a ta+* lop
- else if( hi2 instanceof BinaryOp && ((BinaryOp)hi2).getOp()==OpOp2.MULT
- && hi2.getInput().get(0).getDim2()==1 && hi2.getInput().get(1).getDim2()==1
- && hi2.getParent().size() == 1 //no other consumer than sum
- && !(hi2.getInput().get(0) instanceof BinaryOp && ((BinaryOp)hi2.getInput().get(0)).getOp()==OpOp2.MULT)
- && !(hi2.getInput().get(1) instanceof BinaryOp && ((BinaryOp)hi2.getInput().get(1)).getOp()==OpOp2.MULT))
- {
- baLeft = hi2.getInput().get(0);
- baRight = hi2.getInput().get(1);
- }
-
- //perform actual rewrite (if necessary)
- if( baLeft != null && baRight != null )
- {
- //remove link from parent to diag
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create new operator chain
- ReorgOp trans = HopRewriteUtils.createTranspose(baLeft);
- AggBinaryOp mmult = HopRewriteUtils.createMatrixMultiply(trans, baRight);
-
- UnaryOp cast = new UnaryOp(baLeft.getName(), DataType.SCALAR, ValueType.DOUBLE, OpOp1.CAST_AS_SCALAR, mmult);
- HopRewriteUtils.setOutputParameters(cast, 0, 0, 0, 0, -1);
-
- //rehang new subdag under parent node
- HopRewriteUtils.addChildReference(parent, cast, pos);
- parent.refreshSizeInformation();
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- hi = cast;
-
- LOG.debug("Applied simplifyDotProductSum.");
- }
- }
-
- return hi;
- }
-
- /**
- * Replace SUM(X^2) with a fused SUM_SQ(X) HOP.
- *
- * @param parent Parent HOP for which hi is an input.
- * @param hi Current HOP for potential rewrite.
- * @param pos Position of hi in parent's list of inputs.
- *
- * @return Either hi or the rewritten HOP replacing it.
- *
- * @throws HopsException
- */
- private Hop fuseSumSquared(Hop parent, Hop hi, int pos)
- throws HopsException {
- // if SUM
- if (hi instanceof AggUnaryOp && ((AggUnaryOp) hi).getOp() == AggOp.SUM) {
- Hop sumInput = hi.getInput().get(0);
-
- // if input to SUM is POW(X,2), and no other consumers of the POW(X,2) HOP
- if (sumInput instanceof BinaryOp && ((BinaryOp) sumInput).getOp() == OpOp2.POW
- && sumInput.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp) sumInput.getInput().get(1)) == 2
- && sumInput.getParent().size() == 1) {
- Hop x = sumInput.getInput().get(0);
-
- // if X is NOT a column vector
- if (x.getDim2() > 1) {
- // perform rewrite from SUM(POW(X,2)) to SUM_SQ(X)
- DataType dt = hi.getDataType();
- ValueType vt = hi.getValueType();
- Direction dir = ((AggUnaryOp) hi).getDirection();
- long brlen = hi.getRowsInBlock();
- long bclen = hi.getColsInBlock();
- AggUnaryOp sumSq = new AggUnaryOp("sumSq", dt, vt, AggOp.SUM_SQ, dir, x);
- HopRewriteUtils.setOutputBlocksizes(sumSq, brlen, bclen);
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, sumSq, pos);
-
- // cleanup
- if (hi.getParent().isEmpty())
- HopRewriteUtils.removeAllChildReferences(hi);
- if(sumInput.getParent().isEmpty())
- HopRewriteUtils.removeAllChildReferences(sumInput);
-
- // replace current HOP with new SUM_SQ HOP
- hi = sumSq;
- }
- }
- }
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyEmptyBinaryOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof BinaryOp ) //b(?) X Y
- {
- BinaryOp bop = (BinaryOp) hi;
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- if( left.getDataType()==DataType.MATRIX && right.getDataType()==DataType.MATRIX )
- {
- Hop hnew = null;
-
- //NOTE: these rewrites of binary cell operations need to be aware that right is
- //potentially a vector but the result is of the size of left
-
- boolean notBinaryMV = HopRewriteUtils.isNotMatrixVectorBinaryOperation(bop);
-
- switch( bop.getOp() ){
- //X * Y -> matrix(0,nrow(X),ncol(X));
- case MULT: {
- if( HopRewriteUtils.isEmpty(left) ) //empty left and size known
- hnew = HopRewriteUtils.createDataGenOp(left, left, 0);
- else if( HopRewriteUtils.isEmpty(right) //empty right and right not a vector
- && right.getDim1()>1 && right.getDim2()>1 ) {
- hnew = HopRewriteUtils.createDataGenOp(right, right, 0);
- }
- else if( HopRewriteUtils.isEmpty(right) )//empty right and right potentially a vector
- hnew = HopRewriteUtils.createDataGenOp(left, left, 0);
- break;
- }
- case PLUS: {
- if( HopRewriteUtils.isEmpty(left) && HopRewriteUtils.isEmpty(right) ) //empty left/right and size known
- hnew = HopRewriteUtils.createDataGenOp(left, left, 0);
- else if( HopRewriteUtils.isEmpty(left) && notBinaryMV ) //empty left
- hnew = right;
- else if( HopRewriteUtils.isEmpty(right) ) //empty right
- hnew = left;
- break;
- }
- case MINUS: {
- if( HopRewriteUtils.isEmpty(left) && notBinaryMV ) { //empty left
- HopRewriteUtils.removeChildReference(hi, left);
- HopRewriteUtils.addChildReference(hi, new LiteralOp(0), 0);
- hnew = hi;
- }
- else if( HopRewriteUtils.isEmpty(right) ) //empty and size known
- hnew = left;
- break;
- }
- default:
- hnew = null;
- }
-
- if( hnew != null )
- {
- //remove unnecessary matrix mult
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //create datagen and add it to parent
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- parent.refreshSizeInformation();
-
- hi = hnew;
-
- LOG.debug("Applied simplifyEmptyBinaryOperation");
- }
- }
- }
-
- return hi;
- }
-
- /**
- * This is rewrite tries to reorder minus operators from inputs of matrix
- * multiply to its output because the output is (except for outer products)
- * usually significantly smaller. Furthermore, this rewrite is a precondition
- * for the important hops-lops rewrite of transpose-matrixmult if the transpose
- * is hidden under the minus.
- *
- * NOTE: in this rewrite we need to modify the links to all parents because we
- * remove existing links of subdags and hence affect all consumers.
- *
- * TODO select up or down based on size
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- @SuppressWarnings("unchecked")
- private Hop reorderMinusMatrixMult(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof AggBinaryOp && ((AggBinaryOp)hi).isMatrixMultiply() ) //X%*%Y
- {
- Hop hileft = hi.getInput().get(0);
- Hop hiright = hi.getInput().get(1);
-
- if( hileft instanceof BinaryOp && ((BinaryOp)hileft).getOp()==OpOp2.MINUS //X=-Z
- && hileft.getInput().get(0) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)hileft.getInput().get(0))==0.0 )
- {
- Hop hi2 = hileft.getInput().get(1);
-
- //remove link from matrixmult to minus
- HopRewriteUtils.removeChildReference(hi, hileft);
-
- //get old parents (before creating minus over matrix mult)
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
-
- //create new operators
- BinaryOp minus = new BinaryOp(hi.getName(), hi.getDataType(), hi.getValueType(), OpOp2.MINUS, new LiteralOp(0), hi);
- minus.setRowsInBlock(hi.getRowsInBlock());
- minus.setColsInBlock(hi.getColsInBlock());
-
- //rehang minus under all parents
- for( Hop p : parents ) {
- int ix = HopRewriteUtils.getChildReferencePos(p, hi);
- HopRewriteUtils.removeChildReference(p, hi);
- HopRewriteUtils.addChildReference(p, minus, ix);
- }
-
- //rehang child of minus under matrix mult
- HopRewriteUtils.addChildReference(hi, hi2, 0);
-
- //cleanup if only consumer of minus
- if( hileft.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hileft );
-
- hi = minus;
-
- LOG.debug("Applied reorderMinusMatrixMult");
- }
- else if( hiright instanceof BinaryOp && ((BinaryOp)hiright).getOp()==OpOp2.MINUS //X=-Z
- && hiright.getInput().get(0) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValue((LiteralOp)hiright.getInput().get(0))==0.0 )
- {
- Hop hi2 = hiright.getInput().get(1);
-
- //remove link from matrixmult to minus
- HopRewriteUtils.removeChildReference(hi, hiright);
-
- //get old parents (before creating minus over matrix mult)
- ArrayList<Hop> parents = (ArrayList<Hop>) hi.getParent().clone();
-
- //create new operators
- BinaryOp minus = new BinaryOp(hi.getName(), hi.getDataType(), hi.getValueType(), OpOp2.MINUS, new LiteralOp(0), hi);
- minus.setRowsInBlock(hi.getRowsInBlock());
- minus.setColsInBlock(hi.getColsInBlock());
-
- //rehang minus under all parents
- for( Hop p : parents ) {
- int ix = HopRewriteUtils.getChildReferencePos(p, hi);
- HopRewriteUtils.removeChildReference(p, hi);
- HopRewriteUtils.addChildReference(p, minus, ix);
- }
-
- //rehang child of minus under matrix mult
- HopRewriteUtils.addChildReference(hi, hi2, 1);
-
- //cleanup if only consumer of minus
- if( hiright.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hiright );
-
- hi = minus;
-
- LOG.debug("Applied reorderMinusMatrixMult");
- }
- }
-
- return hi;
- }
-
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifySumMatrixMult(Hop parent, Hop hi, int pos)
- {
- //sum(A%*%B) -> sum(t(colSums(A))*rowSums(B))
- //if not dot product, not applied since aggregate removed
- //if sum not the only consumer, not applied to prevent redundancy
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getOp()==AggOp.SUM //sum
- && ((AggUnaryOp)hi).getDirection() == Direction.RowCol //full aggregate
- && hi.getInput().get(0) instanceof AggBinaryOp //A%*%B
- && (hi.getInput().get(0).getDim1()>1 || hi.getInput().get(0).getDim2()>1) //not dot product
- && hi.getInput().get(0).getParent().size()==1 ) //not multiple consumers of matrix mult
- {
- Hop hi2 = hi.getInput().get(0);
- Hop left = hi2.getInput().get(0);
- Hop right = hi2.getInput().get(1);
-
- //remove link from parent to diag
- HopRewriteUtils.removeChildReference(hi, hi2);
-
- //create new operators
- AggUnaryOp colSum = new AggUnaryOp(left.getName(), left.getDataType(), left.getValueType(), AggOp.SUM, Direction.Col, left);
- colSum.setRowsInBlock(left.getRowsInBlock());
- colSum.setColsInBlock(left.getColsInBlock());
- colSum.refreshSizeInformation();
- ReorgOp trans = HopRewriteUtils.createTranspose(colSum);
- AggUnaryOp rowSum = new AggUnaryOp(right.getName(), right.getDataType(), right.getValueType(), AggOp.SUM, Direction.Row, right);
- rowSum.setRowsInBlock(right.getRowsInBlock());
- rowSum.setColsInBlock(right.getColsInBlock());
- rowSum.refreshSizeInformation();
- BinaryOp mult = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), OpOp2.MULT, trans, rowSum);
- mult.setRowsInBlock(right.getRowsInBlock());
- mult.setColsInBlock(right.getColsInBlock());
- mult.refreshSizeInformation();
-
-
- //rehang new subdag under current node (keep hi intact)
- HopRewriteUtils.addChildReference(hi, mult, 0);
- hi.refreshSizeInformation();
-
- //cleanup if only consumer of intermediate
- if( hi2.getPar
<TRUNCATED>
[27/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Lop.java b/src/main/java/com/ibm/bi/dml/lops/Lop.java
deleted file mode 100644
index 751ca1f..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Lop.java
+++ /dev/null
@@ -1,720 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Base class for all Lops.
- */
-
-public abstract class Lop
-{
-
- public enum Type {
- Data, DataGen, //CP/MR read/write/datagen
- ReBlock, CSVReBlock, //MR reblock operations
- MMCJ, MMRJ, MMTSJ, PMMJ, MapMult, MapMultChain, //MR matrix multiplications
- UnaryCP, UNARY, BinaryCP, Binary, Ternary, //CP/MR unary/binary/ternary
- RangeReIndex, LeftIndex, ZeroOut, //CP/MR indexing
- Aggregate, PartialAggregate, //CP/MR aggregation
- BinUaggChain, UaggOuterChain, //CP/MR aggregation
- TernaryAggregate, //CP ternary-binary aggregates
- Grouping, //MR grouping
- Append, //CP/MR append (column append)
- CombineUnary, CombineBinary, CombineTernary, //MR combine (stitch together)
- CentralMoment, CoVariance, GroupedAgg,
- Transform, DataPartition, RepMat, //CP/MR reorganization, partitioning, replication
- ParameterizedBuiltin, //CP/MR parameterized ops (name/value)
- FunctionCallCP, //CP function calls
- CumulativePartialAggregate, CumulativeSplitAggregate, CumulativeOffsetBinary, //MR cumsum/cumprod/cummin/cummax
- WeightedSquaredLoss, WeightedSigmoid, WeightedDivMM, WeightedCeMM, WeightedUMM,
- SortKeys, PickValues,
- Checkpoint, //Spark persist into storage level
- };
-
- /**
- * Lop types
- */
- public enum SimpleInstType {
- Scalar, Variable, File
- };
-
- public enum VisitStatus {
- DONE, VISITING, NOTVISITED
- }
-
-
- protected static final Log LOG = LogFactory.getLog(Lop.class.getName());
-
- public static final String FILE_SEPARATOR = "/";
- public static final String PROCESS_PREFIX = "_p";
-
- public static final String INSTRUCTION_DELIMITOR = "\u2021"; // "\u002c"; //",";
- public static final String OPERAND_DELIMITOR = "\u00b0"; //\u2021"; //00ea"; //"::#::";
- public static final String VALUETYPE_PREFIX = "\u00b7" ; //":#:";
- public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX; //":#:";
- public static final String LITERAL_PREFIX = VALUETYPE_PREFIX; //":#:";
- public static final String NAME_VALUE_SEPARATOR = "="; // e.g., used in parameterized builtins
- public static final String VARIABLE_NAME_PLACEHOLDER = "##"; //TODO: use in LOPs
- public static final String MATRIX_VAR_NAME_PREFIX = "_mVar";
- public static final String SCALAR_VAR_NAME_PREFIX = "_Var";
-
- // Boolean array to hold the list of nodes(lops) in the DAG that are reachable from this lop.
- private boolean[] reachable = null;
- private DataType _dataType;
- private ValueType _valueType;
-
- private VisitStatus _visited = VisitStatus.NOTVISITED;
-
- protected Lop.Type type;
-
- /**
- * transient indicator
- */
- protected boolean hasTransientParameters = false;
-
- /**
- * handle to all inputs and outputs.
- */
- protected ArrayList<Lop> inputs;
- protected ArrayList<Lop> outputs;
-
- /**
- * refers to #lops whose input is equal to the output produced by this lop.
- * This is used in generating rmvar instructions as soon as the output produced
- * by this lop is consumed. Otherwise, such rmvar instructions are added
- * at the end of program blocks.
- *
- */
- protected int consumerCount;
-
- /**
- * handle to output parameters, dimensions, blocking, etc.
- */
-
- protected OutputParameters outParams = null;
-
- protected LopProperties lps = null;
-
-
- /**
- * Constructor to be invoked by base class.
- *
- * @param t
- */
-
- public Lop(Type t, DataType dt, ValueType vt) {
- type = t;
- _dataType = dt; // data type of the output produced from this LOP
- _valueType = vt; // value type of the output produced from this LOP
- inputs = new ArrayList<Lop>();
- outputs = new ArrayList<Lop>();
- outParams = new OutputParameters();
- lps = new LopProperties();
- }
-
- /**
- * get visit status of node
- *
- * @return
- */
-
- public VisitStatus getVisited() {
- return _visited;
- }
-
- /**
- * set visit status of node
- *
- * @param visited
- */
- public void setVisited(VisitStatus visited) {
- _visited = visited;
- }
-
-
- public boolean[] get_reachable() {
- return reachable;
- }
-
- public boolean[] create_reachable(int size) {
- reachable = new boolean[size];
- return reachable;
- }
-
- /**
- * get data type of the output that is produced by this lop
- *
- * @return
- */
-
- public DataType getDataType() {
- return _dataType;
- }
-
- /**
- * set data type of the output that is produced by this lop
- *
- * @param dt
- */
- public void setDataType(DataType dt) {
- _dataType = dt;
- }
-
- /**
- * get value type of the output that is produced by this lop
- *
- * @return
- */
-
- public ValueType getValueType() {
- return _valueType;
- }
-
- /**
- * set value type of the output that is produced by this lop
- *
- * @param vt
- */
- public void setValueType(ValueType vt) {
- _valueType = vt;
- }
-
-
- /**
- * Method to get Lop type.
- *
- * @return
- */
-
- public Lop.Type getType() {
- return type;
- }
-
- /**
- * Method to get input of Lops
- *
- * @return
- */
- public ArrayList<Lop> getInputs() {
- return inputs;
- }
-
- /**
- * Method to get output of Lops
- *
- * @return
- */
-
- public ArrayList<Lop> getOutputs() {
- return outputs;
- }
-
- /**
- * Method to add input to Lop
- *
- * @param op
- */
-
- public void addInput(Lop op) {
- inputs.add(op);
- }
-
- /**
- * Method to add output to Lop
- *
- * @param op
- */
-
- public void addOutput(Lop op) {
- outputs.add(op);
- }
-
- public int getConsumerCount() {
- return consumerCount;
- }
-
- public void setConsumerCount(int cc) {
- consumerCount = cc;
- }
-
- public int removeConsumer() {
- consumerCount--;
- return consumerCount;
- }
-
- /**
- * Method to have Lops print their state. This is for debugging purposes.
- */
-
- public abstract String toString();
-
- public void resetVisitStatus() {
- if (this.getVisited() == Lop.VisitStatus.NOTVISITED)
- return;
- for (int i = 0; i < this.getInputs().size(); i++) {
- this.getInputs().get(i).resetVisitStatus();
- }
- this.setVisited(Lop.VisitStatus.NOTVISITED);
- }
-
- /**
- * Method to have recursively print state of Lop graph.
- */
-
- public final void printMe() {
- if (LOG.isDebugEnabled()){
- StringBuilder s = new StringBuilder("");
- if (this.getVisited() != VisitStatus.DONE) {
- s.append(getType() + ": " + getID() + "\n" ); // hashCode());
- s.append("Inputs: ");
- for (int i = 0; i < this.getInputs().size(); i++) {
- s.append(" " + this.getInputs().get(i).getID() + " ");
- }
-
- s.append("\n");
- s.append("Outputs: ");
- for (int i = 0; i < this.getOutputs().size(); i++) {
- s.append(" " + this.getOutputs().get(i).getID() + " ");
- }
-
- s.append("\n");
- s.append(this.toString());
- s.append("Begin Line: " + _beginLine + ", Begin Column: " + _beginColumn + ", End Line: " + _endLine + ", End Column: " + _endColumn + "\n");
- s.append("FORMAT:" + this.getOutputParameters().getFormat() + ", rows="
- + this.getOutputParameters().getNumRows() + ", cols=" + this.getOutputParameters().getNumCols()
- + ", Blocked?: " + this.getOutputParameters().isBlocked() + ", rowsInBlock=" +
- this.getOutputParameters().getRowsInBlock() + ", colsInBlock=" +
- this.getOutputParameters().getColsInBlock() + "\n");
- this.setVisited(VisitStatus.DONE);
- s.append("\n");
-
- for (int i = 0; i < this.getInputs().size(); i++) {
- this.getInputs().get(i).printMe();
- }
- }
- LOG.debug(s.toString());
- }
- }
-
- /**
- * Method to return the ID of LOP
- */
- public long getID() {
- return lps.getID();
- }
-
- public int getLevel() {
- return lps.getLevel();
- }
-
- public void setLevel() {
- lps.setLevel(inputs);
- }
-
- /**
- * Method to get the location property of LOP
- *
- * @return
- */
- public ExecLocation getExecLocation() {
- return lps.getExecLocation();
- }
-
- /**
- * Method to get the execution type (CP or MR) of LOP
- *
- * @return
- */
- public ExecType getExecType() {
- return lps.getExecType();
- }
-
- /**
- * Method to get the compatible job type for the LOP
- * @return
- */
-
- public int getCompatibleJobs() {
- return lps.getCompatibleJobs();
- }
-
- /**
- * Method to find if the lop breaks alignment
- */
- public boolean getBreaksAlignment() {
- return lps.getBreaksAlignment();
- }
-
- public boolean getProducesIntermediateOutput() {
- return lps.getProducesIntermediateOutput();
- }
-
- public boolean isAligner()
- {
- return lps.isAligner();
- }
-
- public boolean definesMRJob()
- {
- return lps.getDefinesMRJob();
- }
-
- /**
- * Method to recursively add LOPS to a DAG
- *
- * @param dag
- */
- public final void addToDag(Dag<Lop> dag)
- {
- if( dag.addNode(this) )
- for( Lop l : getInputs() )
- l.addToDag(dag);
- }
-
- /**
- * Method to get output parameters
- *
- * @return
- */
-
- public OutputParameters getOutputParameters() {
- return outParams;
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String input2, String input3, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String input2, String input3, String input4, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String input2, String input3, String input4, String input5, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String input2, String input3, String input4, String input5, String input6, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- public String getInstructions(int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass. Lop Type: " + this.getType());
- }
-
- public String getInstructions(int input_index, int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass. Lop Type: " + this.getType());
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String input2, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String input1, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions(String[] inputs, String[] outputs) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public String getInstructions() throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /** Method should be overridden if needed **/
- public SimpleInstType getSimpleInstructionType() throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for Lops
- ///////////////////////////////////////////////////////////////////////////
- public int _beginLine, _beginColumn;
- public int _endLine, _endColumn;
-
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(int blp, int bcp, int elp, int ecp){
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- //TODO: Leo This might get confused with Rand.getInstructions
- public String getInstructions(String input, String rowl, String rowu,
- String coll, String colu, String leftRowDim,
- String leftColDim, String output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- public String getInstructions(int input, int rowl, int rowu,
- int coll, int colu, int leftRowDim,
- int leftColDim, int output) throws LopsException {
- throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
- }
-
- /**
- * Function that determines if the output of a LOP is defined by a variable or not.
- *
- * @return
- */
- public boolean isVariable() {
- return ( (getExecLocation() == ExecLocation.Data && !((Data)this).isLiteral())
- || !(getExecLocation() == ExecLocation.Data ) );
- }
-
-
-
- /**
- * Method to prepare instruction operand with given parameters.
- *
- * @param label
- * @param dt
- * @param vt
- * @return
- */
- public String prepOperand(String label, DataType dt, ValueType vt) {
- StringBuilder sb = new StringBuilder();
- sb.append(label);
- sb.append(Lop.DATATYPE_PREFIX);
- sb.append(dt);
- sb.append(Lop.VALUETYPE_PREFIX);
- sb.append(vt);
- return sb.toString();
- }
-
- /**
- * Method to prepare instruction operand with given parameters.
- *
- * @param label
- * @param dt
- * @param vt
- * @return
- */
- public String prepOperand(String label, DataType dt, ValueType vt, boolean literal) {
- StringBuilder sb = new StringBuilder();
- sb.append(label);
- sb.append(Lop.DATATYPE_PREFIX);
- sb.append(dt);
- sb.append(Lop.VALUETYPE_PREFIX);
- sb.append(vt);
- sb.append(Lop.LITERAL_PREFIX);
- sb.append(literal);
- return sb.toString();
- }
-
- /**
- * Method to prepare instruction operand with given label. Data type
- * and Value type are derived from Lop's properties.
- *
- * @param label
- * @return
- */
- private String prepOperand(String label) {
- StringBuilder sb = new StringBuilder("");
- sb.append(label);
- sb.append(Lop.DATATYPE_PREFIX);
- sb.append(getDataType());
- sb.append(Lop.VALUETYPE_PREFIX);
- sb.append(getValueType());
- return sb.toString();
- }
-
- public String prepOutputOperand() {
- return prepOperand(getOutputParameters().getLabel());
- }
-
- public String prepOutputOperand(int index) {
- return prepOperand(index+"");
- }
- public String prepOutputOperand(String label) {
- return prepOperand(label);
- }
-
- /**
- * Function to prepare label for scalar inputs while generating instructions.
- * It attaches placeholder suffix and prefixes if the Lop denotes a variable.
- *
- * @return
- */
- public String prepScalarLabel() {
- String ret = getOutputParameters().getLabel();
- if ( isVariable() ){
- ret = Lop.VARIABLE_NAME_PLACEHOLDER + ret + Lop.VARIABLE_NAME_PLACEHOLDER;
- }
- return ret;
- }
-
- /**
- * Function to be used in creating instructions for creating scalar
- * operands. It decides whether or not attach placeholders for instruction
- * patching. Resulting string also encodes if the operand is a literal.
- *
- * For non-literals:
- * Placeholder prefix and suffix need to be attached for Instruction
- * Patching during execution. However, they should NOT be attached IF:
- * - the operand is a literal
- * OR
- * - the execution type is CP. This is because CP runtime has access
- * to symbol table and the instruction encodes sufficient information
- * to determine if an operand is a literal or not.
- *
- * @param et
- * @return
- */
- public String prepScalarOperand(ExecType et, String label) {
- boolean isData = (getExecLocation() == ExecLocation.Data);
- boolean isLiteral = (isData && ((Data)this).isLiteral());
-
- StringBuilder sb = new StringBuilder("");
- if ( et == ExecType.CP || et == ExecType.SPARK || (isData && isLiteral)) {
- sb.append(label);
- }
- else {
- sb.append(Lop.VARIABLE_NAME_PLACEHOLDER);
- sb.append(label);
- sb.append(Lop.VARIABLE_NAME_PLACEHOLDER);
- }
-
- sb.append(Lop.DATATYPE_PREFIX);
- sb.append(getDataType());
- sb.append(Lop.VALUETYPE_PREFIX);
- sb.append(getValueType());
- sb.append(Lop.LITERAL_PREFIX);
- sb.append(isLiteral);
-
- return sb.toString();
- }
-
- public String prepScalarInputOperand(ExecType et) {
- return prepScalarOperand(et, getOutputParameters().getLabel());
- }
-
- public String prepScalarInputOperand(String label) {
- boolean isData = (getExecLocation() == ExecLocation.Data);
- boolean isLiteral = (isData && ((Data)this).isLiteral());
-
- StringBuilder sb = new StringBuilder("");
- sb.append(label);
- sb.append(Lop.DATATYPE_PREFIX);
- sb.append(getDataType());
- sb.append(Lop.VALUETYPE_PREFIX);
- sb.append(getValueType());
- sb.append(Lop.LITERAL_PREFIX);
- sb.append(isLiteral);
-
- return sb.toString();
- }
-
- public String prepInputOperand(int index) {
- return prepInputOperand(index+"");
- }
-
- public String prepInputOperand(String label) {
- DataType dt = getDataType();
- if ( dt == DataType.MATRIX ) {
- return prepOperand(label);
- }
- else {
- return prepScalarInputOperand(label);
- }
- }
-
- /**
- * Method to check if a LOP expects an input from the Distributed Cache.
- * The method in parent class always returns <code>false</code> (default).
- * It must be overridden by individual LOPs that use the cache.
- */
- public boolean usesDistributedCache() {
- return false;
- }
-
- public int[] distributedCacheInputIndex() {
- return new int[]{-1};
- }
-
-
- public boolean hasNonBlockedInputs() {
- for(Lop in : getInputs())
- if(in.getDataType() == DataType.MATRIX && !in.getOutputParameters().isBlocked())
- return true;
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/LopProperties.java b/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
deleted file mode 100644
index 3268e1d..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-
-public class LopProperties
-{
-
- public enum ExecType { CP, CP_FILE, MR, SPARK, INVALID };
- public enum ExecLocation {INVALID, RecordReader, Map, MapOrReduce, MapAndReduce, Reduce, Data, ControlProgram };
-
- // static variable to assign an unique ID to every lop that is created
- private static IDSequence UniqueLopID = null;
-
- static{
- UniqueLopID = new IDSequence();
- }
-
- /**
- * Execution properties for each lop.
- *
- * execLoc = in case execType=MR, where does this lop must run in a job (map, reduce, etc.)
- * compatibleJobs = list of job types in which this lop can be run. It is maintained as a bit vector.
- * breaksAlignment = does this lop alters the keys ?
- * isAligner = is this lop mainly used to reorder/sort/align the keys
- *
- */
- long ID;
- int level;
- ExecType execType;
- ExecLocation execLoc;
- int compatibleJobs;
- boolean breaksAlignment;
- boolean isAligner;
- boolean definesMRJob;
- boolean producesIntermediateOutput;
-
- public LopProperties() {
- ID = UniqueLopID.getNextID();
- execType = ExecType.INVALID;
- execLoc = ExecLocation.INVALID;
- compatibleJobs = JobType.INVALID.getBase();
- breaksAlignment = true;
- isAligner = false;
- definesMRJob = false;
- producesIntermediateOutput = false;
- }
-
- public long getID() { return ID; }
- public int getLevel() { return level; }
- public void setLevel( int l ) { level = l; }
-
- public ExecLocation getExecLocation() {
- return execLoc;
- }
-
- public ExecType getExecType() {
- return execType;
- }
-
- public int getCompatibleJobs() {
- return compatibleJobs;
- }
-
- public boolean getBreaksAlignment() {
- return breaksAlignment;
- }
-
- public boolean getDefinesMRJob() {
- return definesMRJob;
- }
-
- public boolean isAligner()
- {
- return isAligner;
- }
-
- public boolean getProducesIntermediateOutput() {
- return producesIntermediateOutput;
- }
-
- public void setExecLocation(ExecLocation el) {
- execLoc = el;
- }
-
- public void addCompatibility ( JobType jt ) {
- compatibleJobs = compatibleJobs | jt.getBase();
- }
-
- public void removeCompatibility ( JobType jt ) {
- compatibleJobs = compatibleJobs ^ jt.getBase();
- }
-
- public void removeNonPiggybackableJobs() {
- // Remove compatibility with those jobs which do not allow any "other" instructions
- for ( JobType jt : JobType.values()) {
- if(jt.allowsNoOtherInstructions()) {
- compatibleJobs = compatibleJobs ^ jt.getBase();
- }
- }
- }
-
- public void setCompatibleJobs(int cj) {
- compatibleJobs = cj;
- }
-
- public void setDefinesMRJob(boolean dmrj) {
- definesMRJob = dmrj;
- }
-
- public void setBreaksAlignment(boolean ba) {
- breaksAlignment = ba;
- }
-
- public void setAligner(boolean align) {
- isAligner = align;
- }
-
- public void setProducesIntermediateOutput(boolean pio) {
- producesIntermediateOutput = pio;
- }
-
- /*
- * Function to compute the node level in the entire Lops DAG.
- * level(v) = max( levels(v.inputs) ) + 1
- */
- public void setLevel(ArrayList<Lop> inputs) {
- int tmplevel = -1;
- if ( inputs == null || inputs.isEmpty() )
- tmplevel = 0;
- else {
- // find the max level among all inputs
- for(Lop in : inputs) {
- if(tmplevel < in.getLevel() ) {
- tmplevel = in.getLevel();
- }
- }
- // this.level should be one more than the max
- tmplevel = tmplevel+1;
- }
- setLevel(tmplevel);
- }
-
- public void setProperties ( ArrayList<Lop> inputs, ExecType et, ExecLocation el, boolean ba, boolean aligner, boolean definesMR ) {
- execType = et;
- execLoc = el;
- breaksAlignment = ba;
- isAligner = aligner;
- definesMRJob = definesMR;
- setLevel(inputs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/LopsException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/LopsException.java b/src/main/java/com/ibm/bi/dml/lops/LopsException.java
deleted file mode 100644
index 762a3e7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/LopsException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.api.DMLException;
-
-public class LopsException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public LopsException(String message)
- {
- super(message);
- }
-
- public LopsException(Exception e) {
- super(e);
- }
-
- public LopsException(String message, Throwable cause) {
- super(message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMCJ.java b/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
deleted file mode 100644
index dc37667..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class MMCJ extends Lop
-{
-
-
- public enum MMCJType {
- AGG,
- NO_AGG,
- }
-
- //optional attribute for mr exec type
- private MMCJType _type = MMCJType.AGG;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- /**
- * Constructor to perform a cross product operation.
- * @param et
- * @param input
- * @param op
- */
-
- public MMCJ(Lop input1, Lop input2, DataType dt, ValueType vt, MMCJType type, ExecType et)
- {
- super(Lop.Type.MMCJ, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- _type = type;
-
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = true;
- lps.addCompatibility(JobType.MMCJ);
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- this.lps.setProducesIntermediateOutput(true);
- }
- else //if( et == ExecType.SPARK )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- /**
- *
- * @param input1
- * @param input2
- * @param dt
- * @param vt
- * @param aggtype
- * @param et
- */
- public MMCJ(Lop input1, Lop input2, DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- {
- this(input1, input2, dt, vt, MMCJType.NO_AGG, et);
-
- _aggtype = aggtype;
- }
-
-
- @Override
- public String toString() {
-
- return "Operation = MMCJ";
- }
-
- //MR instruction generation
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cpmm" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _type );
-
-
- return sb.toString();
- }
-
- //SPARK instruction generation
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cpmm" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output) );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMRJ.java b/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
deleted file mode 100644
index 8722526..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class MMRJ extends Lop
-{
-
-
- /**
- * Constructor to perform a cross product operation.
- * @param input
- * @param op
- */
-
- public MMRJ(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et)
- {
- //handle inputs and outputs
- super(Lop.Type.MMRJ, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //set basic lop properties based on exec type
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = true;
- lps.addCompatibility(JobType.MMRJ);
- this.lps.setProperties( inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- }
- else //if( et == ExecType.SPARK )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return "Operation = MMRJ";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "rmm" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "rmm" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java b/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
deleted file mode 100644
index d1d3053..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform transpose-identity operation (t(X)%*%X or X%*%t(X)),
- * used to represent CP and MR instruction but in case of MR there is
- * an additional Aggregate at the reducers.
- */
-public class MMTSJ extends Lop
-{
-
-
- public enum MMTSJType {
- NONE,
- LEFT,
- RIGHT;
-
- public boolean isLeft(){
- return (this == LEFT);
- }
- }
-
- private MMTSJType _type = null;
- private int _numThreads = 1;
-
- public MMTSJ(Lop input1, DataType dt, ValueType vt, ExecType et, MMTSJType type)
- {
- this(input1, dt, vt, et, type, -1);
- }
-
- public MMTSJ(Lop input1, DataType dt, ValueType vt, ExecType et, MMTSJType type, int k)
- {
- super(Lop.Type.MMTSJ, dt, vt);
- addInput(input1);
- input1.addOutput(this);
- _type = type;
- _numThreads = k;
-
- boolean breaksAlignment = true; //if result keys (matrix indexes) different
- boolean aligner = false; //if groups multiple inputs by key (e.g., group)
- boolean definesMRJob = (et == ExecType.MR); //if requires its own MR job
- ExecLocation el = (et == ExecType.MR) ? ExecLocation.Map : ExecLocation.ControlProgram;
-
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, et, el, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return "Operation = MMTSJ";
- }
-
- /**
- * MR instruction generation.
- */
- @Override
- public String getInstructions(int input_index1, int output_index)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "tsmm" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
- sb.append( OPERAND_DELIMITOR );
- sb.append( _type );
-
- return sb.toString();
- }
-
- /**
- * CP and Spark instruction generation.
- */
- @Override
- public String getInstructions(String input_index1, String output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( "tsmm" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
- sb.append( OPERAND_DELIMITOR );
- sb.append( _type );
-
- //append degree of parallelism for matrix multiplications
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMZip.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMZip.java b/src/main/java/com/ibm/bi/dml/lops/MMZip.java
deleted file mode 100644
index b8875fb..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMZip.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform zip matrix multiplication
- */
-public class MMZip extends Lop
-{
- private boolean _tRewrite = true;
-
- /**
- *
- * @param input
- * @param op
- */
- public MMZip(Lop input1, Lop input2, DataType dt, ValueType vt, boolean tRewrite, ExecType et)
- {
- //handle inputs and outputs
- super(Lop.Type.MMRJ, dt, vt);
-
- _tRewrite = tRewrite;
-
- addInput(input1);
- addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //only applicable for spark
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
-
- return "Operation = MMZip";
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "zipmm" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( _tRewrite );
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MapMult.java b/src/main/java/com/ibm/bi/dml/lops/MapMult.java
deleted file mode 100644
index e3d4d2e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MapMult.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class MapMult extends Lop
-{
-
- public static final String OPCODE = "mapmm";
-
- public enum CacheType {
- RIGHT,
- RIGHT_PART,
- LEFT,
- LEFT_PART;
-
- public boolean isRightCache(){
- return (this == RIGHT || this == RIGHT_PART);
- }
- }
-
- private CacheType _cacheType = null;
- private boolean _outputEmptyBlocks = true;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- /**
- * Constructor to setup a partial Matrix-Vector Multiplication for MR
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public MapMult(Lop input1, Lop input2, DataType dt, ValueType vt, boolean rightCache, boolean partitioned, boolean emptyBlocks )
- throws LopsException
- {
- super(Lop.Type.MapMult, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup mapmult parameters
- if( rightCache )
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- else
- _cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
- _outputEmptyBlocks = emptyBlocks;
-
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
-
- /**
- * Constructor to setup a partial Matrix-Vector Multiplication for Spark
- *
- * @param input1
- * @param input2
- * @param dt
- * @param vt
- * @param rightCache
- * @param emptyBlocks
- * @param aggregate
- * @param et
- * @throws LopsException
- */
- public MapMult(Lop input1, Lop input2, DataType dt, ValueType vt, boolean rightCache, boolean partitioned, boolean emptyBlocks, SparkAggType aggtype)
- throws LopsException
- {
- super(Lop.Type.MapMult, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup mapmult parameters
- if( rightCache )
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- else
- _cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
- _outputEmptyBlocks = emptyBlocks;
- _aggtype = aggtype;
-
- //setup MR parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- public String toString() {
- return "Operation = MapMM";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- //MR instruction generation
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- //Spark instruction generation
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
-
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( this.prepOutputOperand(output));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(_cacheType);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(_outputEmptyBlocks);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(_aggtype.toString());
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- switch( _cacheType )
- {
- // first input is from distributed cache
- case LEFT:
- case LEFT_PART:
- return new int[]{1};
-
- // second input is from distributed cache
- case RIGHT:
- case RIGHT_PART:
- return new int[]{2};
- }
-
- return new int[]{-1}; //error
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java b/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
deleted file mode 100644
index 8badbc0..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class MapMultChain extends Lop
-{
-
- public static final String OPCODE = "mapmmchain";
- public static final String OPCODE_CP = "mmchain";
-
- public enum ChainType {
- XtXv, //(t(X) %*% (X %*% v))
- XtwXv, //(t(X) %*% (w * (X %*% v)))
- NONE,
- }
-
- private ChainType _chainType = null;
- private int _numThreads = 1;
-
- /**
- * Constructor to setup a map mult chain without weights
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public MapMultChain(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.MapMultChain, dt, vt);
- addInput(input1); //X
- addInput(input2); //v
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup mapmult parameters
- _chainType = ChainType.XtXv;
- setupLopProperties(et);
- }
-
- /**
- * Constructor to setup a map mult chain with weights
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public MapMultChain(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.MapMultChain, dt, vt);
- addInput(input1); //X
- addInput(input2); //w
- addInput(input3); //v
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- //setup mapmult parameters
- _chainType = ChainType.XtwXv;
- setupLopProperties(et);
- }
-
- /**
- *
- * @param et
- */
- private void setupLopProperties( ExecType et )
- {
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-
- public String toString() {
- return "Operation = MapMMChain";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- //MR instruction XtXv
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_chainType);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
- {
- //MR instruction XtwXv
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_chainType);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- //Spark instruction XtXv
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- if( getExecType()==ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_chainType);
-
- //append degree of parallelism for matrix multiplications
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- {
- //Spark instruction XtwXv
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- if( getExecType()==ExecType.CP )
- sb.append(OPCODE_CP);
- else
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_chainType);
-
- //append degree of parallelism for matrix multiplications
- if( getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- if( _chainType == ChainType.XtXv )
- return new int[]{2};
- else if( _chainType == ChainType.XtwXv )
- return new int[]{2,3};
-
- //error
- return new int[]{-1};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java b/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
deleted file mode 100644
index 3772f68..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.HopsException;
-
-/**
- * class to maintain output parameters for a lop.
- *
- */
-
-public class OutputParameters
-{
-
- public enum Format {
- TEXT, BINARY, MM, CSV
- };
-
- private boolean _blocked = true;
- private long _num_rows = -1;
- private long _num_cols = -1;
- private long _nnz = -1;
- private long _num_rows_in_block = -1;
- private long _num_cols_in_block = -1;
- private String _file_name = null;
- private String _file_label = null;
-
- Format matrix_format = Format.BINARY;
-
- public String getFile_name() {
- return _file_name;
- }
-
- public void setFile_name(String fileName) {
- _file_name = fileName;
- }
-
- public String getLabel() {
- return _file_label;
- }
-
- public void setLabel(String label) {
- _file_label = label;
- }
-
- public void setDimensions(long rows, long cols, long rows_per_block, long cols_per_block, long nnz) throws HopsException {
- _num_rows = rows;
- _num_cols = cols;
- _nnz = nnz;
- _num_rows_in_block = rows_per_block;
- _num_cols_in_block = cols_per_block;
-
- if ( _num_rows_in_block == 0 && _num_cols_in_block == 0 ) {
- _blocked = false;
- }
- else if (_num_rows_in_block == -1 && _num_cols_in_block == -1) {
- _blocked = false;
- }
- else if ( _num_rows_in_block > 0 && _num_cols_in_block > 0 ) {
- _blocked = true;
- }
- else {
- throw new HopsException("In OutputParameters Lop, Invalid values for blocking dimensions: [" + _num_rows_in_block + "," + _num_cols_in_block +"].");
- }
- }
-
- public Format getFormat() {
- return matrix_format;
- }
-
- public void setFormat(Format fmt) {
- matrix_format = fmt;
- }
-
- public boolean isBlocked() {
- return _blocked;
- }
-
- public void setBlocked(boolean blocked)
- {
- _blocked = blocked;
- }
-
- public long getNumRows()
- {
- return _num_rows;
- }
-
- public void setNumRows(long rows)
- {
- _num_rows = rows;
- }
-
- public long getNumCols()
- {
- return _num_cols;
- }
-
- public void setNumCols(long cols)
- {
- _num_cols = cols;
- }
-
- public Long getNnz() {
- return _nnz;
- }
-
- public void setNnz(long nnz)
- {
- _nnz = nnz;
- }
-
- public long getRowsInBlock() {
- return _num_rows_in_block;
- }
-
- public void setRowsInBlock(long rows_in_block) {
- _num_rows_in_block = rows_in_block;
- }
-
- public long getColsInBlock() {
- return _num_cols_in_block;
- }
-
- public void setColsInBlock(long cols_in_block) {
- _num_cols_in_block = cols_in_block;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("rows=" + getNumRows() + Lop.VALUETYPE_PREFIX);
- sb.append("cols=" + getNumCols() + Lop.VALUETYPE_PREFIX);
- sb.append("nnz=" + getNnz() + Lop.VALUETYPE_PREFIX);
- sb.append("rowsInBlock=" + getRowsInBlock() + Lop.VALUETYPE_PREFIX);
- sb.append("colsInBlock=" + getColsInBlock() + Lop.VALUETYPE_PREFIX);
- sb.append("isBlockedRepresentation=" + isBlocked() + Lop.VALUETYPE_PREFIX);
- sb.append("format=" + getFormat() + Lop.VALUETYPE_PREFIX);
- sb.append("label=" + getLabel() + Lop.VALUETYPE_PREFIX);
- sb.append("filename=" + getFile_name());
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PMMJ.java b/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
deleted file mode 100644
index c540256..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class PMMJ extends Lop
-{
-
- public static final String OPCODE = "pmm";
-
- public enum CacheType {
- LEFT,
- LEFT_PART;
- }
-
- private CacheType _cacheType = null;
- private boolean _outputEmptyBlocks = true;
- private int _numThreads = 1;
-
- /**
- * Constructor to setup a Permutation Matrix Multiplication
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public PMMJ(Lop pminput, Lop rightinput, Lop nrow, DataType dt, ValueType vt, boolean partitioned, boolean emptyBlocks, ExecType et)
- throws LopsException
- {
- super(Lop.Type.PMMJ, dt, vt);
- addInput(pminput);
- addInput(rightinput);
- addInput(nrow);
- pminput.addOutput(this);
- rightinput.addOutput(this);
- nrow.addOutput(this);
-
- //setup mapmult parameters
- _cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
- _outputEmptyBlocks = emptyBlocks;
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- ExecLocation el = (et == ExecType.MR) ? ExecLocation.Map : ExecLocation.ControlProgram;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, et, el, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return "Operation = PMMJ";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepScalarLabel() );
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheType);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- if( getExecType() == ExecType.SPARK )
- {
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheType);
- }
- else if( getExecType()==ExecType.CP ) {
- //append degree of parallelism
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- //always left cached selection vector
- return new int[]{1};
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PMapMult.java b/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
deleted file mode 100644
index cb12999..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class PMapMult extends Lop
-{
- public static final String OPCODE = "pmapmm";
-
- /**
- *
- * @param input1
- * @param input2
- * @param dt
- * @param vt
- * @param rightCache
- * @param emptyBlocks
- * @param aggregate
- * @param et
- * @throws LopsException
- */
- public PMapMult(Lop input1, Lop input2, DataType dt, ValueType vt)
- throws LopsException
- {
- super(Lop.Type.MapMult, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup MR parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- public String toString() {
- return "Operation = PMapMM";
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- //Spark instruction generation
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java b/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
deleted file mode 100644
index c67a92b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
+++ /dev/null
@@ -1,514 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-
-
-/**
- * Defines a LOP for functions.
- *
- */
-public class ParameterizedBuiltin extends Lop
-{
-
- public enum OperationTypes {
- INVALID, CDF, INVCDF, RMEMPTY, REPLACE, REXPAND,
- PNORM, QNORM, PT, QT, PF, QF, PCHISQ, QCHISQ, PEXP, QEXP,
- TRANSFORM
- };
-
- private OperationTypes _operation;
- private HashMap<String, Lop> _inputParams;
- private boolean _bRmEmptyBC;
-
- /**
- * Creates a new builtin function LOP.
- *
- * @param target
- * target identifier
- * @param params
- * parameter list
- * @param inputParameters
- * list of input LOPs
- * @param function
- * builtin function
- * @param numRows
- * number of resulting rows
- * @param numCols
- * number of resulting columns
- */
- public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.ParameterizedBuiltin, dt, vt);
- _operation = op;
-
- for (Lop lop : paramLops.values()) {
- this.addInput(lop);
- lop.addOutput(this);
- }
-
- _inputParams = paramLops;
-
- /*
- * This lop is executed in control program.
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt, ExecType et)
- throws HopsException
- {
- super(Lop.Type.ParameterizedBuiltin, dt, vt);
- _operation = op;
-
- for (Lop lop : paramLops.values()) {
- this.addInput(lop);
- lop.addOutput(this);
- }
-
- _inputParams = paramLops;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- ExecLocation eloc = null;
-
- if( _operation == OperationTypes.REPLACE && et==ExecType.MR )
- {
- eloc = ExecLocation.MapOrReduce;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- }
- else if( _operation == OperationTypes.RMEMPTY && et==ExecType.MR )
- {
- eloc = ExecLocation.Reduce;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- breaksAlignment=true;
- }
- else if( _operation == OperationTypes.REXPAND && et==ExecType.MR )
- {
- eloc = ExecLocation.MapOrReduce;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- breaksAlignment=true;
- }
- else if ( _operation == OperationTypes.TRANSFORM && et == ExecType.MR ) {
- definesMRJob = true;
- eloc = ExecLocation.MapAndReduce;
- lps.addCompatibility(JobType.TRANSFORM);
- }
- else //executed in CP / CP_FILE / SPARK
- {
- eloc = ExecLocation.ControlProgram;
- lps.addCompatibility(JobType.INVALID);
- }
- lps.setProperties(inputs, et, eloc, breaksAlignment, aligner, definesMRJob);
- }
-
- public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean bRmEmptyBC)
- throws HopsException
- {
- this(paramLops, op, dt, vt, et);
- _bRmEmptyBC = bRmEmptyBC;
- }
-
- public OperationTypes getOp() {
- return _operation;
- }
-
- public int getInputIndex(String name) {
- Lop n = _inputParams.get(name);
- for(int i=0; i<getInputs().size(); i++)
- if(getInputs().get(i) == n)
- return i;
- return -1;
- }
-
- public Lop getNamedInput(String name) {
- return _inputParams.get(name);
- }
-
- @Override
- public String getInstructions(String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- switch(_operation)
- {
- case CDF:
- case INVCDF:
- sb.append( (_operation == OperationTypes.CDF ? "cdf" : "invcdf") );
- sb.append( OPERAND_DELIMITOR );
-
- for ( String s : _inputParams.keySet() )
- {
- sb.append( s );
- sb.append( NAME_VALUE_SEPARATOR );
-
- // get the value/label of the scalar input associated with name "s"
- Lop iLop = _inputParams.get(s);
- sb.append( iLop.prepScalarLabel() );
- sb.append( OPERAND_DELIMITOR );
- }
- break;
-
- case RMEMPTY:
- sb.append("rmempty");
- sb.append(OPERAND_DELIMITOR);
-
- for ( String s : _inputParams.keySet() ) {
-
- sb.append(s);
- sb.append(NAME_VALUE_SEPARATOR);
-
- // get the value/label of the scalar input associated with name "s"
- // (offset and maxdim only apply to exec type spark)
- Lop iLop = _inputParams.get(s);
- if( s.equals( "target") || s.equals( "select") || getExecType()==ExecType.SPARK )
- sb.append( iLop.getOutputParameters().getLabel());
- else
- sb.append( iLop.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
- }
-
- break;
-
- case REPLACE:
- sb.append( "replace" );
- sb.append( OPERAND_DELIMITOR );
-
- for ( String s : _inputParams.keySet() )
- {
- sb.append( s );
- sb.append( NAME_VALUE_SEPARATOR );
-
- // get the value/label of the scalar input associated with name "s"
- Lop iLop = _inputParams.get(s);
- if( s.equals("target") )
- sb.append(iLop.getOutputParameters().getLabel());
- else
- sb.append( iLop.prepScalarLabel() );
- sb.append( OPERAND_DELIMITOR );
- }
- break;
-
- case REXPAND:
- sb.append("rexpand");
- sb.append(OPERAND_DELIMITOR);
-
- for ( String s : _inputParams.keySet() ) {
-
- sb.append(s);
- sb.append(NAME_VALUE_SEPARATOR);
-
- // get the value/label of the scalar input associated with name "s"
- // (offset and maxdim only apply to exec type spark)
- Lop iLop = _inputParams.get(s);
- if( s.equals( "target") || getExecType()==ExecType.SPARK )
- sb.append( iLop.getOutputParameters().getLabel());
- else
- sb.append( iLop.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
- }
-
- break;
-
- case TRANSFORM:
- {
- sb.append("transform");
- sb.append(OPERAND_DELIMITOR);
-
- for ( String s : _inputParams.keySet() ) {
- sb.append(s);
- sb.append(NAME_VALUE_SEPARATOR);
-
- Lop iLop = _inputParams.get(s);
- if( iLop.getDataType() != DataType.SCALAR )
- sb.append( iLop.getOutputParameters().getLabel());
- else
- sb.append( iLop.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
- }
- break;
- }
-
- default:
- throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
- }
-
- if (_operation == OperationTypes.RMEMPTY) {
- sb.append("bRmEmptyBC");
- sb.append(NAME_VALUE_SEPARATOR);
- sb.append( _bRmEmptyBC );
- sb.append(OPERAND_DELIMITOR);
- }
-
- sb.append(this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- switch(_operation)
- {
- case REPLACE:
- {
- sb.append( "replace" );
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop = _inputParams.get("target");
- int pos = getInputs().indexOf(iLop);
- int index = (pos==0)? input_index1 : (pos==1)? input_index2 : input_index3;
- //input_index
- sb.append(prepInputOperand(index));
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop2 = _inputParams.get("pattern");
- sb.append(iLop2.prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop3 = _inputParams.get("replacement");
- sb.append(iLop3.prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- break;
- }
-
- default:
- throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
- }
-
- sb.append( prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- switch(_operation)
- {
- case RMEMPTY:
- {
- sb.append("rmempty");
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop1 = _inputParams.get("target");
- int pos1 = getInputs().indexOf(iLop1);
- int index1 = (pos1==0)? input_index1 : (pos1==1)? input_index2 : (pos1==2)? input_index3 : input_index4;
- sb.append(prepInputOperand(index1));
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop2 = _inputParams.get("offset");
- int pos2 = getInputs().indexOf(iLop2);
- int index2 = (pos2==0)? input_index1 : (pos2==1)? input_index2 : (pos1==2)? input_index3 : input_index4;
- sb.append(prepInputOperand(index2));
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop3 = _inputParams.get("maxdim");
- sb.append( iLop3.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop4 = _inputParams.get("margin");
- sb.append( iLop4.prepScalarLabel() );
-
- sb.append( OPERAND_DELIMITOR );
-
- break;
- }
-
- default:
- throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
- }
-
- sb.append( prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- switch(_operation)
- {
- case REXPAND:
- {
- sb.append("rexpand");
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop1 = _inputParams.get("target");
- int pos1 = getInputs().indexOf(iLop1);
- int index1 = (pos1==0)? input_index1 : (pos1==1)? input_index2 : (pos1==2)? input_index3 : (pos1==3)? input_index4 : input_index5;
- sb.append(prepInputOperand(index1));
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop2 = _inputParams.get("max");
- sb.append( iLop2.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop3 = _inputParams.get("dir");
- sb.append( iLop3.prepScalarLabel() );
-
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop4 = _inputParams.get("cast");
- sb.append( iLop4.prepScalarLabel() );
-
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop5 = _inputParams.get("ignore");
- sb.append( iLop5.prepScalarLabel() );
-
- sb.append( OPERAND_DELIMITOR );
-
- break;
- }
-
- default:
- throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
- }
-
- sb.append( prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- if(_operation== OperationTypes.TRANSFORM)
- {
- // int inputIndex = getInputIndex("target");
-
- sb.append( "transform" );
- sb.append( OPERAND_DELIMITOR );
- Lop iLop = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_DATA);
- sb.append(iLop.prepInputOperand(getInputIndex("target")));
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop2 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXMTD);
- sb.append(iLop2.prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- // either applyTransformPath or transformSpec should be specified
- boolean isApply = false;
- Lop iLop3 = null;
- if ( _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD) != null ) {
- // apply transform
- isApply = true;
- iLop3 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD);
- }
- else {
- iLop3 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXSPEC);
- }
- sb.append(iLop3.prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(isApply);
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop4 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_OUTNAMES);
- if( iLop4 != null )
- {
- sb.append(iLop4.prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
- }
-
- sb.append( prepOutputOperand(output_index));
- }
- //return getTransformInstructions(""+getInputIndex("target"), ""+output_index);
- else
- throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-
- return sb.toString();
- }
-
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(_operation.toString());
-
- if( !getInputs().isEmpty() )
- sb.append("(");
- for (Lop cur : getInputs()) {
- sb.append(cur.toString());
- }
- if( !getInputs().isEmpty() )
- sb.append(") ");
-
- sb.append(" ; num_rows=" + this.getOutputParameters().getNumRows());
- sb.append(" ; num_cols=" + this.getOutputParameters().getNumCols());
- sb.append(" ; format=" + this.getOutputParameters().getFormat());
- sb.append(" ; blocked=" + this.getOutputParameters().isBlocked());
- return sb.toString();
- }
-
-}
[28/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CombineUnary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CombineUnary.java b/src/main/java/com/ibm/bi/dml/lops/CombineUnary.java
deleted file mode 100644
index da843e1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CombineUnary.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to represent an combine operation -- used ONLY in the context of sort.
- */
-
-public class CombineUnary extends Lop
-{
-
-
-
- /**
- * @param input - input lop
- * @param op - operation type
- */
-
- public CombineUnary(Lop input1, DataType dt, ValueType vt)
- {
- super(Lop.Type.CombineUnary, dt, vt);
- this.addInput(input1);
- input1.addOutput(this);
-
- /*
- * This lop gets piggybacked into, and can ONLY be executed in SORT_KEYS job
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.SORT);
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
-
- public String toString()
- {
- return "combineunary";
- }
-
- @Override
- public String getInstructions(int input_index1, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "combineunary" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- public static CombineUnary constructCombineLop(Lop input1,
- DataType dt, ValueType vt) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
-
- for (Lop lop : set1) {
- if ( lop.type == Lop.Type.CombineUnary ) {
- return (CombineUnary)lop;
- }
- }
-
- CombineUnary comn = new CombineUnary(input1, dt, vt);
- comn.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndLine());
- return comn;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CumulativeOffsetBinary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CumulativeOffsetBinary.java b/src/main/java/com/ibm/bi/dml/lops/CumulativeOffsetBinary.java
deleted file mode 100644
index 290824a..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CumulativeOffsetBinary.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- *
- *
- */
-public class CumulativeOffsetBinary extends Lop
-{
-
- private OperationTypes _op;
- private double _initValue = 0;
-
- public CumulativeOffsetBinary(Lop data, Lop offsets, DataType dt, ValueType vt, OperationTypes op, ExecType et)
- throws LopsException
- {
- super(Lop.Type.CumulativeOffsetBinary, dt, vt);
- checkSupportedOperations(op);
- _op = op;
-
- init(data, offsets, dt, vt, et);
- }
-
- public CumulativeOffsetBinary(Lop data, Lop offsets, DataType dt, ValueType vt, double init, OperationTypes op, ExecType et)
- throws LopsException
- {
- super(Lop.Type.CumulativeOffsetBinary, dt, vt);
- checkSupportedOperations(op);
- _op = op;
-
- //in case of Spark, CumulativeOffset includes CumulativeSplit and hence needs the init value
- _initValue = init;
-
- init(data, offsets, dt, vt, et);
- }
-
- /**
- *
- * @param input
- * @param dt
- * @param vt
- * @param et
- */
- private void init(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et)
- {
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties(inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob);
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- public String toString() {
- return "CumulativeOffsetBinary";
- }
-
- /**
- *
- * @param op
- * @throws LopsException
- */
- private void checkSupportedOperations(OperationTypes op)
- throws LopsException
- {
- //sanity check for supported aggregates
- if( !(op == OperationTypes.KahanSum || op == OperationTypes.Product ||
- op == OperationTypes.Min || op == OperationTypes.Max) )
- {
- throw new LopsException("Unsupported aggregate operation type: "+op);
- }
- }
-
- private String getOpcode() {
- switch( _op ) {
- case KahanSum: return "bcumoffk+";
- case Product: return "bcumoff*";
- case Min: return "bcumoffmin";
- case Max: return "bcumoffmax";
- default: return null;
- }
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index) );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output) );
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _initValue );
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CumulativePartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CumulativePartialAggregate.java b/src/main/java/com/ibm/bi/dml/lops/CumulativePartialAggregate.java
deleted file mode 100644
index 7590316..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CumulativePartialAggregate.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- *
- *
- */
-public class CumulativePartialAggregate extends Lop
-{
-
- private OperationTypes _op;
-
- public CumulativePartialAggregate(Lop input, DataType dt, ValueType vt, OperationTypes op, ExecType et)
- throws LopsException
- {
- super(Lop.Type.CumulativePartialAggregate, dt, vt);
-
- //sanity check for supported aggregates
- if( !(op == OperationTypes.KahanSum || op == OperationTypes.Product ||
- op == OperationTypes.Min || op == OperationTypes.Max) )
- {
- throw new LopsException("Unsupported aggregate operation type: "+op);
- }
- _op = op;
-
- init(input, dt, vt, et);
- }
-
- /**
- *
- * @param input
- * @param dt
- * @param vt
- * @param et
- */
- private void init(Lop input, DataType dt, ValueType vt, ExecType et)
- {
- this.addInput(input);
- input.addOutput(this);
-
- if( et == ExecType.MR )
- {
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else //Spark/CP
- {
- //setup Spark parameters
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return "CumulativePartialAggregate";
- }
-
- /**
- *
- * @return
- */
- private String getOpcode()
- {
- switch( _op ) {
- case KahanSum: return "ucumack+";
- case Product: return "ucumac*";
- case Min: return "ucumacmin";
- case Max: return "ucumacmax";
- default: return null;
- }
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index) );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output) );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CumulativeSplitAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CumulativeSplitAggregate.java b/src/main/java/com/ibm/bi/dml/lops/CumulativeSplitAggregate.java
deleted file mode 100644
index fc22c5e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CumulativeSplitAggregate.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- *
- */
-public class CumulativeSplitAggregate extends Lop
-{
-
- private double _initValue = 0;
-
- public CumulativeSplitAggregate(Lop input, DataType dt, ValueType vt, double init)
- throws LopsException
- {
- super(Lop.Type.CumulativeSplitAggregate, dt, vt);
- _initValue = init;
- init(input, dt, vt, ExecType.MR);
- }
-
- /**
- *
- * @param input
- * @param dt
- * @param vt
- * @param et
- */
- private void init(Lop input, DataType dt, ValueType vt, ExecType et) {
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
-
- public String toString() {
- return "CumulativeSplitAggregate";
- }
-
- private String getOpcode() {
- return "ucumsplit";
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( String.valueOf(_initValue) );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Data.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Data.java b/src/main/java/com/ibm/bi/dml/lops/Data.java
deleted file mode 100644
index eca031d..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Data.java
+++ /dev/null
@@ -1,619 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-
-/**
- * Lop to represent data objects. Data objects represent matrices, vectors,
- * variables, literals. Can be for both input and output.
- */
-
-public class Data extends Lop
-{
-
- public enum OperationTypes {READ,WRITE};
-
- FileFormatTypes formatType = FileFormatTypes.BINARY;
- OperationTypes operation;
- boolean literal_var = false;
- boolean transient_var = false;
-
- private HashMap<String, Lop> _inputParams;
-
- /**
- * Method to create literal LOPs.
- *
- * @param vt
- * @param literalValue
- * @return
- * @throws LopsException
- */
- public static Data createLiteralLop(ValueType vt, String literalValue) throws LopsException {
- // All literals have default format type of TEXT
- return new Data(OperationTypes.READ, null, null, null, literalValue, DataType.SCALAR, vt, false, FileFormatTypes.BINARY);
- }
-
- /**
- * Constructor to setup read or write LOP
- * In case of write: <code>input</code> must be provided. This will always be added as the first element in <code>input</code> array.
- * For literals: this function is invoked through the static method <code>createLiteralLop</code>.
- *
- * @param op
- * @param input
- * @param inputParametersLops
- * @param name
- * @param literal
- * @param dt
- * @param vt
- * @param isTransient
- * @throws LopsException
- */
- public Data(Data.OperationTypes op, Lop input, HashMap<String, Lop>
- inputParametersLops, String name, String literal, DataType dt, ValueType vt, boolean isTransient, FileFormatTypes fmt) throws LopsException
- {
- super(Lop.Type.Data, dt, vt);
-
- operation = op;
-
- transient_var = isTransient;
-
- // Either <code>name</code> or <code>literal</code> can be non-null.
- if(literal != null){
- if (transient_var )
- throw new LopsException("Invalid parameter values while setting up a Data LOP -- transient flag is invalid for a literal.");
- literal_var = true;
- this.getOutputParameters().setLabel(literal);
- }
- else if(name != null) {
- if ( transient_var )
- this.getOutputParameters().setLabel(name); // tvar+name
- else
- this.getOutputParameters().setLabel("p"+op+name);
- }
- else {
- throw new LopsException("Invalid parameter values while setting up a Data LOP -- the lop must have either literal value or a name.");
- }
-
- // WRITE operation must have an input Lops, we always put this
- // input Lops as the first element of WRITE input. The parameters of
- // WRITE operation are then put as the following input elements.
- if(input != null && operation == OperationTypes.WRITE)
- {
- this.addInput(input);
- input.addOutput(this);
- }
-
- _inputParams = inputParametersLops;
-
- if ( _inputParams != null ) {
- for (Lop lop : inputParametersLops.values()) {
- this.addInput(lop);
- lop.addOutput(this);
- }
- if ( inputParametersLops.get(DataExpression.IO_FILENAME)!= null
- && inputParametersLops.get(DataExpression.IO_FILENAME) instanceof Data )
- {
- OutputParameters outParams = ((Data)inputParametersLops.get(DataExpression.IO_FILENAME)).getOutputParameters();
- String fName = outParams.getLabel();
- this.getOutputParameters().setFile_name(fName);
- }
- }
-
- setFileFormatAndProperties(fmt);
- }
-
- private void setLopProperties() {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
-
- if ( getFileFormatType() == FileFormatTypes.CSV ) {
- Lop input = getInputs().get(0);
- // If the input is data transform, then csv write can be piggybacked onto TRANSFORM job.
- // Otherwise, the input must be converted to csv format via WriteCSV MR job.
- if ( input instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)input).getOp() == com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
- lps.addCompatibility(JobType.TRANSFORM);
- definesMRJob = false;
- }
- else {
- lps.addCompatibility(JobType.CSV_WRITE);
- definesMRJob = true;
- }
- }
- else {
- /*
- * This lop can be executed in all job types except for RAND, PARTITION.
- * RAND: because all inputs must be of type random.
- */
- lps.addCompatibility(JobType.ANY);
- // reads are not compatible with RAND because RAND must have all inputs that are random
- if ( operation == OperationTypes.READ )
- lps.removeCompatibility(JobType.DATAGEN);
- else if ( operation == OperationTypes.WRITE ) {
- // WRITE lops are not compatible with jobs that produce an
- // intermediate output, which MUST be consumed by other subsequent lops
- lps.removeCompatibility(JobType.MMCJ);
- // If at all, SORT job can only write in BinaryBlock format
- if(this.getDataType() == DataType.MATRIX && formatType != FileFormatTypes.BINARY)
- lps.removeCompatibility(JobType.SORT);
- lps.removeCompatibility(JobType.COMBINE);
- }
- }
- // ExecType is invalid for Data lop
- this.lps.setProperties ( inputs, ExecType.INVALID, ExecLocation.Data, breaksAlignment, aligner, definesMRJob );
- }
-
- /**
- * Data-Lop-specific method to set the execution type for persistent write.
- * TODO: split lops into MR/CP lop.
- *
- * @param et
- */
- public void setExecType( ExecType et )
- {
- lps.execType = et;
- }
-
- /**
- * Method to set format types for input, output files.
- * @param type
- * @throws LopsException
- */
- public void setFileFormatAndProperties(FileFormatTypes type) throws LopsException
- {
- this.formatType = type ;
- if(type == FileFormatTypes.BINARY)
- this.outParams.setFormat(Format.BINARY) ;
- else if(type == FileFormatTypes.TEXT)
- this.outParams.setFormat(Format.TEXT) ;
- else if (type == FileFormatTypes.MM)
- this.outParams.setFormat(Format.MM);
- else if (type == FileFormatTypes.CSV )
- this.outParams.setFormat(Format.CSV);
- else
- throw new LopsException("Unexpected format: " + type);
- setLopProperties();
- }
-
- /**
- * method to get format type for input, output files.
- * @return
- */
- public FileFormatTypes getFileFormatType()
- {
- return formatType ;
- }
-
- @Override
- public String toString() {
-
- return getID() + ":" + "File_Name: " + this.getOutputParameters().getFile_name() + " " +
- "Label: " + this.getOutputParameters().getLabel() + " " + "Operation: = " + operation + " " +
- "Format: " + this.outParams.getFormat() + " Datatype: " + getDataType() + " Valuetype: " + getValueType() + " num_rows = " + this.getOutputParameters().getNumRows() + " num_cols = " +
- this.getOutputParameters().getNumCols();
- }
-
- /**
- * method to get operation type, i.e. read/write.
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- /**
- * method to get inputParams
- * @return
- */
- public HashMap<String, Lop> getInputParams(){
- return _inputParams;
- }
-
- public Lop getNamedInputLop(String name) {
- return _inputParams.get(name);
- }
-
- /**
- * method to check if this data lop represents a literal.
- * @return
- */
-
- public boolean isLiteral()
- {
- return literal_var;
-
- }
-
- public boolean getBooleanValue() throws LopsException {
- if(literal_var) {
- return Boolean.parseBoolean(getOutputParameters().getLabel());
- }
- else
- throw new LopsException("Cannot obtain the value of a non-literal variable at compile time.");
- }
-
- public double getDoubleValue() throws LopsException {
- if(literal_var) {
- return Double.parseDouble(getOutputParameters().getLabel());
- }
- else
- throw new LopsException("Cannot obtain the value of a non-literal variable at compile time.");
- }
-
- public long getLongValue() throws LopsException {
- if(literal_var) {
- ValueType vt = getValueType();
- switch(vt) {
- case INT:
- return Long.parseLong(getOutputParameters().getLabel());
- case DOUBLE:
- return (long) Double.parseDouble(getOutputParameters().getLabel());
-
- default:
- throw new LopsException("Encountered a non-numeric value " + (vt) + ", while a numeric value is expected.");
- }
- }
- else
- throw new LopsException("Can not obtain the value of a non-literal variable at compile time.");
- }
-
- public String getStringValue() throws LopsException {
- if(literal_var) {
- return getOutputParameters().getLabel();
- }
- else
- throw new LopsException("Cannot obtain the value of a non-literal variable at compile time.");
- }
-
- /**
- * Method to check if this represents a transient variable.
- * @return
- */
- public boolean isTransient()
- {
- return transient_var;
- }
-
- /**
- *
- * @return
- */
- public boolean isPersistentWrite()
- {
- return operation == OperationTypes.WRITE && !transient_var;
- }
-
- /**
- * Method to generate appropriate MR write instructions.
- * Explicit write instructions are generated only in case of external file formats
- * (e.g., CSV) except for MatrixMarket. MM format is overridden by TextCell, instead.
- *
- */
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException {
-
- OutputParameters oparams = getOutputParameters();
-
- if ( operation != OperationTypes.WRITE )
- throw new LopsException("This method should only be executed for generating MR Write instructions.");
-
- if ( oparams.getFormat() != Format.CSV )
- throw new LopsException("MR Write instructions can not be generated for the output format: " + oparams.getFormat() );
-
- StringBuilder sb = new StringBuilder();
- sb.append( "MR" );
- sb.append( OPERAND_DELIMITOR );
-
- // Generate opcode based on the output format
- if ( oparams.getFormat() == Format.CSV )
- sb.append( "csvwrite" );
- else
- throw new LopsException("MR Write instructions can not be generated for the output format: " + oparams.getFormat() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- // Attach format-specific properties
- if(oparams.getFormat() == Format.CSV) {
- Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Data sparseLop = (Data) getNamedInputLop(DataExpression.DELIM_SPARSE);
-
- if (headerLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_HAS_HEADER_ROW
- + " must be a literal.");
- if (delimLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_DELIMITER
- + " must be a literal.");
- if (sparseLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_SPARSE
- + " must be a literal.");
-
- sb.append(OPERAND_DELIMITOR);
- sb.append(headerLop.getBooleanValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(delimLop.getStringValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(sparseLop.getBooleanValue());
- }
- else {
- throw new LopsException("MR Write instructions can not be generated for the output format: " + oparams.getFormat() );
- }
-
- return sb.toString();
- }
-
- /**
- * Method to get CP instructions for reading/writing scalars and matrices from/to HDFS.
- * This method generates CP read/write instructions.
- */
- @Override
- public String getInstructions(String input1, String input2)
- throws LopsException
- {
- if ( getOutputParameters().getFile_name() == null && operation == OperationTypes.READ )
- throw new LopsException(this.printErrorLocation() + "Data.getInstructions(): Exepecting a SCALAR data type, encountered " + getDataType());
-
- StringBuilder sb = new StringBuilder();
- if(this.getExecType() == ExecType.SPARK)
- sb.append( "SPARK" );
- else
- sb.append( "CP" );
- sb.append( OPERAND_DELIMITOR );
- if ( operation == OperationTypes.READ )
- {
- sb.append( "read" );
- sb.append( OPERAND_DELIMITOR );
- sb.append ( this.prepInputOperand(input1) );
- }
- else if ( operation == OperationTypes.WRITE)
- {
- sb.append( "write" );
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(0).prepInputOperand(input1) );
- }
- else
- throw new LopsException(this.printErrorLocation() + "In Data Lop, Unknown operation: " + operation);
-
- sb.append( OPERAND_DELIMITOR );
- Lop fnameLop = _inputParams.get(DataExpression.IO_FILENAME);
- boolean literal = (fnameLop instanceof Data && ((Data)fnameLop).isLiteral());
- sb.append ( prepOperand(input2, DataType.SCALAR, ValueType.STRING, literal) );
-
- // attach outputInfo in case of matrices
- OutputParameters oparams = getOutputParameters();
- if ( operation == OperationTypes.WRITE ) {
- sb.append( OPERAND_DELIMITOR );
- String fmt = "";
- if ( getDataType() == DataType.MATRIX ) {
- if ( oparams.getFormat() == Format.MM )
- fmt = "matrixmarket";
- else if (oparams.getFormat() == Format.TEXT)
- fmt = "textcell";
- else if (oparams.getFormat() == Format.CSV)
- fmt = "csv";
- else if ( oparams.getFormat() == Format.BINARY ){
- if ( oparams.getRowsInBlock() > 0 || oparams.getColsInBlock() > 0 )
- fmt = "binaryblock";
- else
- fmt = "binarycell" ;
- }
- else {
- throw new LopsException("Unexpected format: " + oparams.getFormat());
- }
- }
- else {
- // scalars will always be written in text format
- fmt = "textcell";
- }
-
- sb.append( prepOperand(fmt, DataType.SCALAR, ValueType.STRING, true));
-
- if(oparams.getFormat() == Format.CSV) {
- Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Data sparseLop = (Data) getNamedInputLop(DataExpression.DELIM_SPARSE);
-
- if (headerLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_HAS_HEADER_ROW
- + " must be a literal for a seq operation.");
- if (delimLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_DELIMITER
- + " must be a literal for a seq operation.");
- if (sparseLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_SPARSE
- + " must be a literal for a seq operation.");
-
- sb.append(OPERAND_DELIMITOR);
- sb.append(headerLop.getBooleanValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(delimLop.getStringValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(sparseLop.getBooleanValue());
-
- if ( this.getExecType() == ExecType.SPARK )
- {
- boolean isInputMatrixBlock = true;
- Lop input = getInputs().get(0);
- if ( input instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)input).getOp() == ParameterizedBuiltin.OperationTypes.TRANSFORM ) {
- // in the case of transform input, the input will be Text strings insteadof MatrixBlocks
- // This information is used to have correct class information while accessing RDDs from the symbol table
- isInputMatrixBlock = false;
- }
- sb.append(OPERAND_DELIMITOR);
- sb.append(isInputMatrixBlock);
- }
- }
-
- }
-
- return sb.toString();
- }
-
- /**
- * Method to generate createvar instruction that updates symbol table with metadata, hdfsfile name, etc.
- *
- * @throws LopsException
- */
- public String getInstructions() throws LopsException {
- return getCreateVarInstructions(getOutputParameters().getFile_name(), getOutputParameters().getLabel());
- }
-
- public String getInstructions(String outputFileName) throws LopsException {
- return getCreateVarInstructions(outputFileName, getOutputParameters().getLabel() );
- }
-
- public String getCreateVarInstructions(String outputFileName, String outputLabel) throws LopsException {
- if ( getDataType() == DataType.MATRIX || getDataType() == DataType.FRAME ) {
-
- if ( isTransient() )
- throw new LopsException("getInstructions() should not be called for transient nodes.");
-
- OutputParameters oparams = getOutputParameters();
- String fmt = "";
- // TODO: following logic should change once we LOPs encode key-value-class information.
- if ( oparams.getFormat() == Format.TEXT )
- fmt = "textcell";
- else if ( oparams.getFormat() == Format.MM )
- fmt = "matrixmarket";
- else if ( oparams.getFormat() == Format.CSV )
- fmt = "csv";
- else {
- if ( oparams.getRowsInBlock() > 0 || oparams.getColsInBlock() > 0 )
- fmt = "binaryblock";
- else
- fmt = "binarycell";
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( "CP" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( "createvar" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( outputLabel );
- sb.append( OPERAND_DELIMITOR );
- sb.append( outputFileName );
- sb.append( OPERAND_DELIMITOR );
- sb.append( false );
- sb.append( OPERAND_DELIMITOR ); // only persistent reads come here!
- sb.append( fmt );
- sb.append( OPERAND_DELIMITOR );
- sb.append( oparams.getNumRows() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( oparams.getNumCols() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( oparams.getRowsInBlock() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( oparams.getColsInBlock() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( oparams.getNnz() );
-
- /* Format-specific properties */
- if ( oparams.getFormat() == Format.CSV ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( createVarCSVHelper() );
- }
-
- return sb.toString();
- }
- else {
- throw new LopsException(this.printErrorLocation() + "In Data Lop, Unexpected data type " + getDataType());
- }
- }
-
- /**
- * Helper function that attaches CSV format-specific properties to createvar instruction.
- * The set of properties that are attached for a READ operation is different from that for a WRITE operation.
- *
- * @return
- * @throws LopsException
- */
- private String createVarCSVHelper() throws LopsException {
- StringBuilder sb = new StringBuilder();
- if ( operation == OperationTypes.READ ) {
- Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Data fillLop = (Data) getNamedInputLop(DataExpression.DELIM_FILL);
- Data fillValueLop = (Data) getNamedInputLop(DataExpression.DELIM_FILL_VALUE);
- Data naLop = (Data) getNamedInputLop(DataExpression.DELIM_NA_STRINGS);
-
- sb.append(headerLop.getBooleanValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(delimLop.getStringValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(fillLop.getBooleanValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(fillValueLop.getDoubleValue());
- if ( naLop != null ) {
- sb.append(OPERAND_DELIMITOR);
- sb.append(naLop.getStringValue());
- }
- }
- else { // (operation == OperationTypes.WRITE)
- Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Data sparseLop = (Data) getNamedInputLop(DataExpression.DELIM_SPARSE);
-
- if (headerLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_HAS_HEADER_ROW
- + " must be a literal for a seq operation.");
- if (delimLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_DELIMITER
- + " must be a literal for a seq operation.");
- if (sparseLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_SPARSE
- + " must be a literal for a seq operation.");
-
- sb.append(headerLop.getBooleanValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(delimLop.getStringValue());
- sb.append(OPERAND_DELIMITOR);
- sb.append(sparseLop.getBooleanValue());
- }
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/DataGen.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/DataGen.java b/src/main/java/com/ibm/bi/dml/lops/DataGen.java
deleted file mode 100644
index 9cea440..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/DataGen.java
+++ /dev/null
@@ -1,558 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * <p>Defines a LOP that generates data.</p>
- */
-public class DataGen extends Lop
-{
-
- public static final String RAND_OPCODE = "rand"; //rand
- public static final String SEQ_OPCODE = "seq"; //sequence
- public static final String SINIT_OPCODE = "sinit"; //string initialize
- public static final String SAMPLE_OPCODE = "sample"; //sample.int
-
- private int _numThreads = 1;
-
- /** base dir for rand input */
- private String baseDir;
-
- private HashMap<String, Lop> _inputParams;
- DataGenMethod method;
-
- /**
- * <p>Creates a new Rand-LOP. The target identifier has to hold the dimensions of the new random object.</p>
- *
- * @param id target identifier
- * @param inputParameterLops Lops of input parameters
- * @param baseDir base dir for runtime
- * @param dt Data type
- * @param vt Value type
- * @param ExecType Execution type
- */
- public DataGen(DataGenMethod mthd, DataIdentifier id, HashMap<String, Lop>
- inputParametersLops, String baseDir, DataType dt, ValueType vt, ExecType et) throws LopsException
- {
- super(Type.DataGen, dt, vt);
- method = mthd;
-
- for (Lop lop : inputParametersLops.values()) {
- this.addInput(lop);
- lop.addOutput(this);
- }
-
- _inputParams = inputParametersLops;
-
- init(id, baseDir, et);
- }
-
- public DataGenMethod getDataGenMethod() {
- return method;
- }
-
- public void init(DataIdentifier id, String baseDir, ExecType et)
- {
- this.getOutputParameters().setFormat(Format.BINARY);
- this.getOutputParameters().setBlocked(true);
- this.getOutputParameters().setNumRows(id.getDim1());
- this.getOutputParameters().setNumCols(id.getDim2());
- this.getOutputParameters().setNnz(-1);
- this.getOutputParameters().setRowsInBlock(id.getRowsInBlock());
- this.getOutputParameters().setColsInBlock(id.getColumnsInBlock());
-
- this.baseDir = baseDir;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob;
-
- if ( et == ExecType.MR ) {
- definesMRJob = true;
- lps.addCompatibility(JobType.DATAGEN);
- this.lps.setProperties( inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else {
- definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- /**
- * Function to generate CP/SP instructions for data generation via Rand and Seq.
- * Since DataGen Lop can have many inputs, ONLY the output variable name is
- * passed from piggybacking as the function argument <code>output</code>.
- */
- @Override
- public String getInstructions(String output)
- throws LopsException
- {
- switch( method )
- {
- case RAND:
- return getCPInstruction_Rand(output);
- case SINIT:
- return getCPInstruction_SInit(output);
- case SEQ:
- return getCPInstruction_Seq(output);
- case SAMPLE:
- return getCPInstruction_Sample(output);
-
- default:
- throw new LopsException("Unknown data generation method: " + method);
- }
- }
-
- @Override
- public String getInstructions(int inputIndex, int outputIndex)
- throws LopsException
- {
- switch(method) {
- case RAND:
- return getMRInstruction_Rand(inputIndex, outputIndex);
- case SEQ:
- return getMRInstruction_Seq(inputIndex, outputIndex);
-
- default:
- throw new LopsException("Unknown data generation method: " + method);
- }
- }
-
- /**
- * Private method that generates CP Instruction for Rand.
- * @param output
- * @return
- * @throws LopsException
- */
- private String getCPInstruction_Rand(String output)
- throws LopsException
- {
- //sanity checks
- if ( method != DataGenMethod.RAND )
- throw new LopsException("Invalid instruction generation for data generation method " + method);
- if( getInputs().size() != DataExpression.RAND_VALID_PARAM_NAMES.length ) {
- throw new LopsException(printErrorLocation() + "Invalid number of operands ("
- + getInputs().size() + ") for a Rand operation");
- }
-
- StringBuilder sb = new StringBuilder( );
-
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append(RAND_OPCODE);
- sb.append(OPERAND_DELIMITOR);
-
- Lop iLop = _inputParams.get(DataExpression.RAND_ROWS.toString());
- sb.append(iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_COLS.toString());
- sb.append(iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- sb.append(String.valueOf(getOutputParameters().getRowsInBlock()));
- sb.append(OPERAND_DELIMITOR);
-
- sb.append(String.valueOf(getOutputParameters().getColsInBlock()));
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_MIN.toString());
- sb.append(iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_MAX.toString());
- sb.append(iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_SPARSITY.toString()); //no variable support
- if (iLop.isVariable())
- throw new LopsException(printErrorLocation()
- + "Parameter " + DataExpression.RAND_SPARSITY
- + " must be a literal for a Rand operation.");
- sb.append(iLop.getOutputParameters().getLabel());
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_SEED.toString());
- sb.append(iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- if ( getExecType() == ExecType.MR ) {
- sb.append(baseDir);
- sb.append(OPERAND_DELIMITOR);
- }
-
- iLop = _inputParams.get(DataExpression.RAND_PDF.toString()); //no variable support
- if (iLop.isVariable())
- throw new LopsException(printErrorLocation()
- + "Parameter " + DataExpression.RAND_PDF
- + " must be a literal for a Rand operation.");
- sb.append(iLop.getOutputParameters().getLabel());
- sb.append(OPERAND_DELIMITOR);
-
- iLop = _inputParams.get(DataExpression.RAND_LAMBDA.toString()); //no variable support
- sb.append(iLop == null ? "" : iLop.prepScalarLabel());
- sb.append(OPERAND_DELIMITOR);
-
- if( getExecType() == ExecType.CP ) {
- //append degree of parallelism
- sb.append( _numThreads );
- sb.append( OPERAND_DELIMITOR );
- }
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- *
- * @param output
- * @return
- * @throws LopsException
- */
- private String getCPInstruction_SInit(String output)
- throws LopsException
- {
- if ( method != DataGenMethod.SINIT )
- throw new LopsException("Invalid instruction generation for data generation method " + method);
-
- //prepare instruction parameters
- Lop iLop = _inputParams.get(DataExpression.RAND_ROWS.toString());
- String rowsString = iLop.prepScalarLabel();
-
- iLop = _inputParams.get(DataExpression.RAND_COLS.toString());
- String colsString = iLop.prepScalarLabel();
-
- String rowsInBlockString = String.valueOf(this
- .getOutputParameters().getRowsInBlock());
- String colsInBlockString = String.valueOf(this
- .getOutputParameters().getColsInBlock());
-
- iLop = _inputParams.get(DataExpression.RAND_MIN.toString());
- String minString = iLop.getOutputParameters().getLabel();
- if (iLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.RAND_MIN
- + " must be a literal for a Rand operation.");
-
- //generate instruction
- StringBuilder sb = new StringBuilder( );
- ExecType et = getExecType();
-
- sb.append( et );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append(SINIT_OPCODE);
- sb.append(OPERAND_DELIMITOR);
- sb.append(rowsString);
- sb.append(OPERAND_DELIMITOR);
- sb.append(colsString);
- sb.append(OPERAND_DELIMITOR);
- sb.append(rowsInBlockString);
- sb.append(OPERAND_DELIMITOR);
- sb.append(colsInBlockString);
- sb.append(OPERAND_DELIMITOR);
- sb.append(minString);
- sb.append(OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- *
- * @param output
- * @return
- * @throws LopsException
- */
- private String getCPInstruction_Sample(String output)
- throws LopsException
- {
- if ( method != DataGenMethod.SAMPLE )
- throw new LopsException("Invalid instruction generation for data generation method " + method);
-
- //prepare instruction parameters
- Lop lsize = _inputParams.get(DataExpression.RAND_ROWS.toString());
- String size = lsize.prepScalarLabel();
-
- Lop lrange = _inputParams.get(DataExpression.RAND_MAX.toString());
- String range = lrange.prepScalarLabel();
-
- Lop lreplace = _inputParams.get(DataExpression.RAND_PDF.toString());
- String replace = lreplace.prepScalarLabel();
-
- Lop lseed = _inputParams.get(DataExpression.RAND_SEED.toString());
- String seed = lseed.prepScalarLabel();
-
- String rowsInBlockString = String.valueOf(this.getOutputParameters().getRowsInBlock());
- String colsInBlockString = String.valueOf(this.getOutputParameters().getColsInBlock());
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "sample" );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( range );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( size );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( replace );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( seed );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( rowsInBlockString );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( colsInBlockString );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output) );
-
- return sb.toString();
- }
-
- /**
- * Private method that generates CP Instruction for Seq.
- * @param output
- * @return
- * @throws LopsException
- */
- private String getCPInstruction_Seq(String output) throws LopsException {
- if ( method != DataGenMethod.SEQ )
- throw new LopsException("Invalid instruction generation for data generation method " + method);
-
- StringBuilder sb = new StringBuilder( );
- ExecType et = getExecType();
- sb.append( et );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- Lop iLop = null;
-
- iLop = _inputParams.get(Statement.SEQ_FROM.toString());
- String fromString = iLop.prepScalarLabel();
-
- iLop = _inputParams.get(Statement.SEQ_TO.toString());
- String toString = iLop.prepScalarLabel();
-
- iLop = _inputParams.get(Statement.SEQ_INCR.toString());
- String incrString = iLop.prepScalarLabel();
-
- String rowsString = String.valueOf(this.getOutputParameters().getNumRows());
- String colsString = String.valueOf(this.getOutputParameters().getNumCols());
- String rowsInBlockString = String.valueOf(this.getOutputParameters().getRowsInBlock());
- String colsInBlockString = String.valueOf(this.getOutputParameters().getColsInBlock());
-
- sb.append( DataGen.SEQ_OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsInBlockString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsInBlockString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( fromString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( toString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( incrString );
- sb.append( OPERAND_DELIMITOR );
- if ( et == ExecType.MR ) {
- sb.append( baseDir );
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Private method to generate MR instruction for Rand.
- *
- * @param input_index
- * @param output_index
- * @return
- * @throws LopsException
- */
- private String getMRInstruction_Rand(int inputIndex, int outputIndex)
- throws LopsException
- {
- //sanity checks
- if( getInputs().size() != DataExpression.RAND_VALID_PARAM_NAMES.length) {
- throw new LopsException(printErrorLocation() + "Invalid number of operands ("
- + getInputs().size() + ") for a Rand operation");
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( RAND_OPCODE );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( inputIndex );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( outputIndex );
- sb.append( OPERAND_DELIMITOR );
-
- Lop iLop = _inputParams.get(DataExpression.RAND_ROWS.toString());
- sb.append( iLop.prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_COLS.toString());
- sb.append( iLop.prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( String.valueOf(getOutputParameters().getRowsInBlock()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( String.valueOf(getOutputParameters().getColsInBlock()) );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_MIN.toString());
- sb.append( iLop.prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_MAX.toString());
- sb.append( iLop.prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_SPARSITY.toString()); //no variable support
- if (iLop.isVariable())
- throw new LopsException(this.printErrorLocation() + "Parameter "
- + DataExpression.RAND_SPARSITY + " must be a literal for a Rand operation.");
- sb.append( iLop.getOutputParameters().getLabel() );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_SEED.toString());
- sb.append( iLop.prepScalarLabel() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( baseDir );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_PDF.toString()); //no variable support
- if (iLop.isVariable())
- throw new LopsException(this.printErrorLocation() + "Parameter "
- + DataExpression.RAND_PDF + " must be a literal for a Rand operation.");
- sb.append( iLop.getOutputParameters().getLabel() );
- sb.append( OPERAND_DELIMITOR );
-
- iLop = _inputParams.get(DataExpression.RAND_LAMBDA.toString()); //no variable support
- sb.append( iLop == null ? "" : iLop.prepScalarLabel() );
-
- return sb.toString();
- }
-
- /**
- * Private method to generate MR instruction for Seq.
- *
- * @param input_index
- * @param output_index
- * @return
- * @throws LopsException
- */
- private String getMRInstruction_Seq(int inputIndex, int outputIndex) throws LopsException {
- StringBuilder sb = new StringBuilder();
-
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- Lop iLop = null;
- iLop = _inputParams.get(Statement.SEQ_FROM.toString());
- String fromString = iLop.getOutputParameters().getLabel();
- if ( (iLop.getExecLocation() == ExecLocation.Data &&
- !((Data)iLop).isLiteral()) || !(iLop.getExecLocation() == ExecLocation.Data ))
- fromString = Lop.VARIABLE_NAME_PLACEHOLDER + fromString + Lop.VARIABLE_NAME_PLACEHOLDER;
-
- iLop = _inputParams.get(Statement.SEQ_TO.toString());
- String toString = iLop.getOutputParameters().getLabel();
- if ( iLop.getExecLocation() == ExecLocation.Data
- && !((Data)iLop).isLiteral() || !(iLop.getExecLocation() == ExecLocation.Data ) )
- toString = Lop.VARIABLE_NAME_PLACEHOLDER + toString + Lop.VARIABLE_NAME_PLACEHOLDER;
-
- iLop = _inputParams.get(Statement.SEQ_INCR.toString());
- String incrString = iLop.getOutputParameters().getLabel();
- if ( iLop.getExecLocation() == ExecLocation.Data
- && !((Data)iLop).isLiteral() || !(iLop.getExecLocation() == ExecLocation.Data ) )
- incrString = Lop.VARIABLE_NAME_PLACEHOLDER + incrString + Lop.VARIABLE_NAME_PLACEHOLDER;
-
- String rowsString = String.valueOf(this.getOutputParameters().getNumRows());
- String colsString = String.valueOf(this.getOutputParameters().getNumCols());
- String rowsInBlockString = String.valueOf(this.getOutputParameters().getRowsInBlock());
- String colsInBlockString = String.valueOf(this.getOutputParameters().getColsInBlock());
-
- sb.append( DataGen.SEQ_OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( inputIndex );
- sb.append( OPERAND_DELIMITOR );
- sb.append( outputIndex );
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsInBlockString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsInBlockString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( fromString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( toString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( incrString );
- sb.append( OPERAND_DELIMITOR );
- sb.append( baseDir );
-
- return sb.toString();
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append(method.toString());
- sb.append(" ; num_rows=" + this.getOutputParameters().getNumRows());
- sb.append(" ; num_cols=" + this.getOutputParameters().getNumCols());
- sb.append(" ; nnz=" + this.getOutputParameters().getNnz());
- sb.append(" ; num_rows_per_block=" + this.getOutputParameters().getRowsInBlock());
- sb.append(" ; num_cols_per_block=" + this.getOutputParameters().getColsInBlock());
- sb.append(" ; format=" + this.getOutputParameters().getFormat());
- sb.append(" ; blocked=" + this.getOutputParameters().isBlocked());
- sb.append(" ; dir=" + this.baseDir);
- return sb.toString();
- }
-
- public void setNumThreads(int k) {
- _numThreads = k;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/DataPartition.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/DataPartition.java b/src/main/java/com/ibm/bi/dml/lops/DataPartition.java
deleted file mode 100644
index 1434611..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/DataPartition.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-
-
-/**
- * Lop to perform data partitioning.
- */
-public class DataPartition extends Lop
-{
-
- public static final String OPCODE = "partition";
-
- private PDataPartitionFormat _pformat = null;
-
- public DataPartition(Lop input, DataType dt, ValueType vt, ExecType et, PDataPartitionFormat pformat)
- throws LopsException
- {
- super(Lop.Type.DataPartition, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- _pformat = pformat;
-
- //setup lop properties
- ExecLocation eloc = (et==ExecType.MR)? ExecLocation.MapAndReduce : ExecLocation.ControlProgram;
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = (et==ExecType.MR);
- lps.addCompatibility(JobType.DATA_PARTITION);
- lps.setProperties( inputs, et, eloc, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
-
- return "DataPartition";
- }
-
- //CP instruction generation
- @Override
- public String getInstructions(String input_index, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( input_index );
- sb.append( DATATYPE_PREFIX );
- sb.append( getInputs().get(0).getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getInputs().get(0).getValueType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( output_index );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _pformat.toString() );
-
- return sb.toString();
- }
-
- //MR instruction generation
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( input_index );
- sb.append( DATATYPE_PREFIX );
- sb.append( getInputs().get(0).getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getInputs().get(0).getValueType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( output_index );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _pformat.toString() );
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/FunctionCallCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/FunctionCallCP.java b/src/main/java/com/ibm/bi/dml/lops/FunctionCallCP.java
deleted file mode 100644
index e5ee9be..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/FunctionCallCP.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- *
- */
-public class FunctionCallCP extends Lop
-{
- private String _fnamespace;
- private String _fname;
- private String[] _outputs;
- private ArrayList<Lop> _outputLops = null;
-
- public FunctionCallCP(ArrayList<Lop> inputs, String fnamespace, String fname, String[] outputs, ArrayList<Hop> outputHops) throws HopsException, LopsException {
- this(inputs, fnamespace, fname, outputs);
- if(outputHops != null) {
- _outputLops = new ArrayList<Lop>();
- for(Hop h : outputHops) {
- _outputLops.add( h.constructLops() );
- }
- }
- }
-
- public FunctionCallCP(ArrayList<Lop> inputs, String fnamespace, String fname, String[] outputs)
- {
- super(Lop.Type.FunctionCallCP, DataType.UNKNOWN, ValueType.UNKNOWN);
- //note: data scalar in order to prevent generation of redundant createvar, rmvar
-
- _fnamespace = fnamespace;
- _fname = fname;
- _outputs = outputs;
-
- //wire inputs
- for( Lop in : inputs )
- {
- addInput( in );
- in.addOutput( this );
- }
-
- //lop properties: always in CP
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- public ArrayList<Lop> getFunctionOutputs() {
- return _outputLops;
- }
-
- @Override
- public String toString() {
- return "function call: " + DMLProgram.constructFunctionKey(_fnamespace, _fname);
- }
-
- private String getInstructionsMultipleReturnBuiltins(String[] inputs, String[] outputs) {
- StringBuilder sb = new StringBuilder();
- sb.append("CP");
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_fname.toLowerCase());
-
- for(int i=0; i< inputs.length; i++) {
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(i).prepInputOperand(inputs[i]) );
- }
-
- for(int i=0; i< _outputs.length; i++) {
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_outputs[i]);
- }
-
- return sb.toString();
- }
-
- /**
- * Method to generate instructions for external functions as well as builtin functions with multiple returns.
- * Builtin functions have their namespace set to DMLProgram.INTERNAL_NAMESPACE ("_internal").
- */
- @Override
- public String getInstructions(String[] inputs, String[] outputs) throws LopsException
- {
- // Handle internal builtin functions
- if (_fnamespace.equalsIgnoreCase(DMLProgram.INTERNAL_NAMESPACE) ) {
- return getInstructionsMultipleReturnBuiltins(inputs, outputs);
- }
-
- //Instruction format extFunct:::[FUNCTION NAMESPACE]:::[FUNCTION NAME]:::[num input params]:::[num output params]:::[list of delimited input params ]:::[list of delimited ouput params]
- //These are the "bound names" for the inputs / outputs. For example, out1 = ns::foo(in1, in2) yields
- //extFunct:::ns:::foo:::2:::1:::in1:::in2:::out1
- //NOTE: we have to append full input operand information to distinguish literals from variables w/ equal name
-
- StringBuilder inst = new StringBuilder();
-
- inst.append("CP");
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append("extfunct");
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append(_fnamespace);
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append(_fname);
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append(inputs.length);
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append(_outputs.length);
-
- for(int i=0; i<inputs.length; i++) {
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append( getInputs().get(i).prepInputOperand(inputs[i]) );
- }
-
- // TODO function output dataops (phase 3) - take 'outputs' into account
- for( String out : _outputs ) {
- inst.append(Lop.OPERAND_DELIMITOR);
- inst.append(out);
- }
-
- return inst.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Group.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Group.java b/src/main/java/com/ibm/bi/dml/lops/Group.java
deleted file mode 100644
index 7af0ca1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Group.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-/**
- * Lop to represent a grouping operation.
- */
-
-public class Group extends Lop
-{
-
-
-
- public enum OperationTypes {Sort};
-
- OperationTypes operation;
-
- /**
- * Constructor to create a grouping operation.
- * @param input
- * @param op
- */
-
- public Group(Lop input, Group.OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.Grouping, dt, vt);
- operation = op;
- this.addInput(input);
- input.addOutput(this);
-
- /*
- * This lop can be executed in only in GMR and RAND.
- * MMCJ, REBLOCK, and PARTITION themselves has location MapAndReduce.
- */
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
-
- boolean breaksAlignment = false;
- boolean aligner = true;
- boolean definesMRJob = true;
-
- this.lps.setProperties ( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString()
- {
- //return "Group " + "Operation: " + operation;
- return "Operation: " + operation;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/GroupedAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/GroupedAggregate.java b/src/main/java/com/ibm/bi/dml/lops/GroupedAggregate.java
deleted file mode 100644
index 974844b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/GroupedAggregate.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform grouped aggregates
- *
- */
-public class GroupedAggregate extends Lop
-{
-
- private HashMap<String, Lop> _inputParams;
- private static final String opcode = "groupedagg";
- public static final String COMBINEDINPUT = "combinedinput";
-
- /**
- * Constructor to perform grouped aggregate.
- * inputParameterLops <- parameters required to compute different aggregates (hashmap)
- * "combinedinput" -- actual data
- * "function" -- aggregate function
- */
-
- private void init(HashMap<String, Lop> inputParameterLops,
- DataType dt, ValueType vt, ExecType et) {
- if ( et == ExecType.MR ) {
- /*
- * Inputs to ParameterizedBuiltinOp can be in an arbitrary order. However,
- * piggybacking (Dag.java:getAggAndOtherInstructions()) expects the first
- * input to be the data (named as "combinedinput") on which the aggregate
- * needs to be computed. Make sure that "combinedinput" is the first input
- * to GroupedAggregate lop.
- */
- this.addInput(inputParameterLops.get(COMBINEDINPUT));
- inputParameterLops.get(COMBINEDINPUT).addOutput(this);
-
- // process remaining parameters
- for ( Entry<String, Lop> e : inputParameterLops.entrySet() ) {
- String k = e.getKey();
- Lop lop = e.getValue();
- if ( !k.equalsIgnoreCase(COMBINEDINPUT) ) {
- this.addInput(lop);
- lop.addOutput(this);
- }
- }
-
- _inputParams = inputParameterLops;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- lps.addCompatibility(JobType.GROUPED_AGG);
- this.lps.setProperties(inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- }
- else {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- // First, add inputs corresponding to "target" and "groups"
- this.addInput(inputParameterLops.get(Statement.GAGG_TARGET));
- inputParameterLops.get(Statement.GAGG_TARGET).addOutput(this);
- this.addInput(inputParameterLops.get(Statement.GAGG_GROUPS));
- inputParameterLops.get(Statement.GAGG_GROUPS).addOutput(this);
-
- // process remaining parameters
- for ( Entry<String, Lop> e : inputParameterLops.entrySet() ) {
- String k = e.getKey();
- Lop lop = e.getValue();
- if ( !k.equalsIgnoreCase(Statement.GAGG_TARGET) && !k.equalsIgnoreCase(Statement.GAGG_GROUPS) ) {
- this.addInput(lop);
- lop.addOutput(this);
- }
- }
- _inputParams = inputParameterLops;
-
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public GroupedAggregate(
- HashMap<String, Lop> inputParameterLops,
- DataType dt, ValueType vt) {
- this(inputParameterLops, dt, vt, ExecType.MR);
- }
-
- public GroupedAggregate(
- HashMap<String, Lop> inputParameterLops,
- DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.GroupedAgg, dt, vt);
- init(inputParameterLops, dt, vt, et);
- }
-
- @Override
- public String toString() {
-
- return "Operation = GroupedAggregate";
- }
-
- /**
- * Function to generate CP Grouped Aggregate Instructions.
- *
- */
- @Override
- public String getInstructions(String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( opcode );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- if ( _inputParams.get(Statement.GAGG_TARGET) == null || _inputParams.get(Statement.GAGG_GROUPS) == null || _inputParams.get("fn") == null )
- throw new LopsException(this.printErrorLocation() + "Invalid parameters to groupedAggregate -- \"target\", \"groups\", \"fn\" must be provided");
-
- String targetVar = _inputParams.get(Statement.GAGG_TARGET).getOutputParameters().getLabel();
- String groupsVar = _inputParams.get(Statement.GAGG_GROUPS).getOutputParameters().getLabel();
-
- sb.append( Statement.GAGG_TARGET );
- sb.append( Lop.NAME_VALUE_SEPARATOR );
- sb.append( targetVar );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( Statement.GAGG_GROUPS );
- sb.append( Lop.NAME_VALUE_SEPARATOR );
- sb.append( groupsVar );
-
- if ( _inputParams.get(Statement.GAGG_WEIGHTS) != null )
- {
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( Statement.GAGG_WEIGHTS );
- sb.append( Lop.NAME_VALUE_SEPARATOR );
- sb.append( _inputParams.get(Statement.GAGG_WEIGHTS).getOutputParameters().getLabel() );
- }
-
- // Process all other name=value parameters, which are scalars
- String name, valueString;
- Lop value;
- for(Entry<String, Lop> e : _inputParams.entrySet()) {
- name = e.getKey();
- if ( !name.equalsIgnoreCase(Statement.GAGG_TARGET) && !name.equalsIgnoreCase(Statement.GAGG_GROUPS) && !name.equalsIgnoreCase(Statement.GAGG_WEIGHTS) ) {
- value = e.getValue();
- valueString = value.prepScalarLabel();
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( name );
- sb.append( Lop.NAME_VALUE_SEPARATOR );
- sb.append( valueString );
- }
- }
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /*@Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "groupedagg" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( input1 );
- sb.append( DATATYPE_PREFIX );
- sb.append( getInputs().get(0).getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getInputs().get(0).getValueType() );
- sb.append( input2 );
- sb.append( DATATYPE_PREFIX );
- sb.append( getInputs().get(1).getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getInputs().get(1).getValueType() );
- sb.append( output );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
-
- return sb.toString();
- }*/
-
- @Override
- public String getInstructions(int input_index, int output_index)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( opcode );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index));
-
- // get the aggregate function
- sb.append( OPERAND_DELIMITOR );
- Lop funcLop = _inputParams.get(Statement.GAGG_FN);
- sb.append( funcLop.prepScalarInputOperand(getExecType()));
-
- // get the "optional" parameters
- if ( _inputParams.get(Statement.GAGG_FN_CM_ORDER) != null ) {
- sb.append( OPERAND_DELIMITOR );
- Lop orderLop = _inputParams.get(Statement.GAGG_FN_CM_ORDER);
- sb.append( orderLop.prepScalarInputOperand(getExecType()));
- }
-
- // add output_index to instruction
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/LeftIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/LeftIndex.java b/src/main/java/com/ibm/bi/dml/lops/LeftIndex.java
deleted file mode 100644
index d81b250..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/LeftIndex.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class LeftIndex extends Lop
-{
-
- /**
- * Constructor to setup a LeftIndexing operation.
- * Example: A[i:j, k:l] = B;
- *
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
-
- private void init(Lop lhsMatrix, Lop rhsMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, ExecType et) throws LopsException {
- /*
- * A[i:j, k:l] = B;
- * B -> rhsMatrix
- * A -> lhsMatrix
- * i,j -> rowL, rowU
- * k,l -> colL, colU
- */
- this.addInput(lhsMatrix);
- this.addInput(rhsMatrix);
- this.addInput(rowL);
- this.addInput(rowU);
- this.addInput(colL);
- this.addInput(colU);
-
- lhsMatrix.addOutput(this);
- rhsMatrix.addOutput(this);
- rowL.addOutput(this);
- rowU.addOutput(this);
- colL.addOutput(this);
- colU.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- throw new LopsException(this.printErrorLocation() + "LeftIndexing lop is undefined for MR runtime");
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public LeftIndex(
- Lop lhsInput, Lop rhsInput, Lop rowL, Lop rowU, Lop colL, Lop colU, DataType dt, ValueType vt, ExecType et)
- throws LopsException {
- super(Lop.Type.LeftIndex, dt, vt);
- init(lhsInput, rhsInput, rowL, rowU, colL, colU, et);
- }
-
- boolean isBroadcast = false;
- public LeftIndex(
- Lop lhsInput, Lop rhsInput, Lop rowL, Lop rowU, Lop colL, Lop colU, DataType dt, ValueType vt, ExecType et, boolean isBroadcast)
- throws LopsException {
- super(Lop.Type.LeftIndex, dt, vt);
- this.isBroadcast = isBroadcast;
- init(lhsInput, rhsInput, rowL, rowU, colL, colU, et);
- }
-
- private String getOpcode() {
- if(isBroadcast)
- return "mapLeftIndex";
- else
- return "leftIndex";
- }
-
- @Override
- public String getInstructions(String lhsInput, String rhsInput, String rowl, String rowu, String coll, String colu, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(lhsInput));
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(1).getDataType() == DataType.SCALAR ) {
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- }
- else {
- sb.append( getInputs().get(1).prepInputOperand(rhsInput));
- }
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append(getInputs().get(2).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- sb.append(getInputs().get(3).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append(getInputs().get(4).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- sb.append(getInputs().get(5).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
-/* @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index)
- throws LopsException {
-
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
-
- String rowl = this.getInputs().get(1).getOutputParameters().getLabel();
- if (this.getInputs().get(1).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(1)).isLiteral())
- rowl = "##" + rowl + "##";
- String rowu = this.getInputs().get(2).getOutputParameters().getLabel();
- if (this.getInputs().get(2).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(2)).isLiteral())
- rowu = "##" + rowu + "##";
- String coll = this.getInputs().get(3).getOutputParameters().getLabel();
- if (this.getInputs().get(3).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(3)).isLiteral())
- coll = "##" + coll + "##";
- String colu = this.getInputs().get(4).getOutputParameters().getLabel();
- if (this.getInputs().get(4).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(4)).isLiteral())
- colu = "##" + colu + "##";
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, Integer.toString(output_index));
- }
-*/
- @Override
- public String toString() {
- return "leftIndex";
- }
-
-}
[11/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
deleted file mode 100644
index 2805d9c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.python;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.v4.runtime.Token;
-
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-
-
-public class PydmlSyntacticValidatorHelper {
-
- private CustomDmlErrorListener _errorListener = null;
- public PydmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
- this._errorListener = errorListener;
- }
- public void notifyErrorListeners(String message, int line, int charPositionInLine) {
- this._errorListener.validationError(line, charPositionInLine, message);
- }
-
- public void notifyErrorListeners(String message, Token op) {
- this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public String getCurrentFileName() {
- return _errorListener.peekFileName();
- }
-
- // Returns list of two elements <namespace, function names>, else null
- public ArrayList<String> getQualifiedNames(String fullyQualifiedFunctionName) {
- String [] fnNames = fullyQualifiedFunctionName.split("\\."); // instead of ::
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else
- return null;
-
- ArrayList<String> retVal = new ArrayList<String>();
- retVal.add(namespace);
- retVal.add(functionName);
- return retVal;
- }
-
-// public static void setInfoForArithmeticOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// try {
-// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// com.ibm.bi.dml.parser.Expression.BinaryOp bop = com.ibm.bi.dml.parser.Expression.getBinaryOp(opStr);
-// current = new com.ibm.bi.dml.parser.BinaryExpression(bop);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-// catch(Exception e) {
-// System.out.println("In setInfoForArithmeticOp>>");
-// e.printStackTrace();
-// }
-// }
-
-// public static void setInfoForBooleanOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// com.ibm.bi.dml.parser.Expression.BooleanOp bop = com.ibm.bi.dml.parser.Expression.getBooleanOp(opStr);
-// current = new com.ibm.bi.dml.parser.BooleanExpression(bop);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-
- public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
- String functionName = ctx.name.getText().replaceAll(" ", "").trim();
- if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + ".write") == 0) {
- return validateBuiltinWriteFunction(ctx);
- }
- return true;
- }
-
- private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
-
- return true;
- }
-
- public ArrayList<com.ibm.bi.dml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<com.ibm.bi.dml.parser.ParameterExpression> retVal = new ArrayList<com.ibm.bi.dml.parser.ParameterExpression>();
- for(ParameterizedExpressionContext ctx : paramExprs) {
- String paramName = null;
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- paramName = ctx.paramName.getText();
- }
- com.ibm.bi.dml.parser.ParameterExpression myArg = new com.ibm.bi.dml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
- retVal.add(myArg);
- }
- return retVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java b/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
deleted file mode 100644
index 82417a5..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.ibm.bi.dml.parser.python;
-
-/**
- * This class exists solely to prevent compiler warnings.
- *
- * <p>
- * The ExpressionInfo and StatementInfo classes are shared among both parsers
- * (R-like and Python-like dialects), and Antlr-generated code assumes that
- * these classes are present in the parser's namespace.
- */
-class StatementInfo extends com.ibm.bi.dml.parser.antlr4.StatementInfo {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
deleted file mode 100644
index 2bee04c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * This exception should be thrown to flag runtime errors -- DML equivalent to java.lang.RuntimeException.
- */
-public class DMLRuntimeException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLRuntimeException(String string) {
- super(string);
- }
-
- public DMLRuntimeException(Exception e) {
- super(e);
- }
-
- public DMLRuntimeException(String string, Exception ex){
- super(string,ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
deleted file mode 100644
index 8caa3e4..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-
-/**
- * This exception should be thrown to flag DML Script errors.
- */
-public class DMLScriptException extends DMLRuntimeException
-{
-
- private static final long serialVersionUID = 1L;
-
- //prevent string concatenation of classname w/ stop message
- private DMLScriptException(Exception e) {
- super(e);
- }
-
- private DMLScriptException(String string, Exception ex){
- super(string,ex);
- }
-
- /**
- * This is the only valid constructor for DMLScriptException.
- *
- * @param string
- */
- public DMLScriptException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
deleted file mode 100644
index c968a03..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-import com.ibm.bi.dml.api.DMLException;
-
-public class DMLUnsupportedOperationException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLUnsupportedOperationException(String string) {
- super(string);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
deleted file mode 100644
index b48472c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ /dev/null
@@ -1,1064 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.FunctionParameter.FunctionParameterType;
-import com.ibm.bi.dml.udf.BinaryObject;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
-
-public class ExternalFunctionProgramBlock extends FunctionProgramBlock
-{
-
- protected static final IDSequence _idSeq = new IDSequence();
-
- protected String _baseDir = null;
-
- ArrayList<Instruction> block2CellInst;
- ArrayList<Instruction> cell2BlockInst;
-
- // holds other key value parameters specified in function declaration
- protected HashMap<String, String> _otherParams;
-
- protected HashMap<String, String> _unblockedFileNames;
- protected HashMap<String, String> _blockedFileNames;
-
- protected long _runID = -1; //ID for block of statements
-
- /**
- * Constructor that also provides otherParams that are needed for external
- * functions. Remaining parameters will just be passed to constructor for
- * function program block.
- *
- * @param eFuncStat
- * @throws DMLRuntimeException
- */
- protected ExternalFunctionProgramBlock(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- String baseDir) throws DMLRuntimeException
- {
- super(prog, inputParams, outputParams);
- _baseDir = baseDir;
-
- //NOTE: no need to setup nimble queue for CP external functions
- }
-
- public ExternalFunctionProgramBlock(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> otherParams,
- String baseDir) throws DMLRuntimeException {
-
- super(prog, inputParams, outputParams);
- _baseDir = baseDir;
-
- // copy other params
- _otherParams = new HashMap<String, String>();
- _otherParams.putAll(otherParams);
-
- _unblockedFileNames = new HashMap<String, String>();
- _blockedFileNames = new HashMap<String, String>();
-
- // generate instructions
- createInstructions();
- }
-
- private void changeTmpInput( long id, ExecutionContext ec )
- {
- ArrayList<DataIdentifier> inputParams = getInputParams();
- block2CellInst = getBlock2CellInstructions(inputParams, _unblockedFileNames);
-
- //post processing FUNCTION PATCH
- for( String var : _skipInReblock )
- {
- Data dat = ec.getVariable(var);
- if( dat instanceof MatrixObject )
- _unblockedFileNames.put(var, ((MatrixObject)dat).getFileName());
- }
- }
-
- /**
- * It is necessary to change the local temporary files as only file handles are passed out
- * by the external function program block.
- *
- *
- * @param id
- */
- private void changeTmpOutput( long id )
- {
- ArrayList<DataIdentifier> outputParams = getOutputParams();
- cell2BlockInst = getCell2BlockInstructions(outputParams, _blockedFileNames);
- }
-
- /**
- *
- * @return
- */
- public String getBaseDir()
- {
- return _baseDir;
- }
-
- /**
- * Method to be invoked to execute instructions for the external function
- * invocation
- * @throws DMLRuntimeException
- */
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException
- {
- _runID = _idSeq.getNextID();
-
- changeTmpInput( _runID, ec );
- changeTmpOutput( _runID );
-
- // export input variables to HDFS (see RunMRJobs)
- ArrayList<DataIdentifier> inputParams = null;
-
- try {
- inputParams = getInputParams();
- for(DataIdentifier di : inputParams ) {
- Data d = ec.getVariable(di.getName());
- if ( d.getDataType() == DataType.MATRIX ) {
- MatrixObject inputObj = (MatrixObject) d;
- inputObj.exportData();
- }
- }
- }
- catch (Exception e){
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
- }
-
- // convert block to cell
- if( block2CellInst != null )
- {
- ArrayList<Instruction> tempInst = new ArrayList<Instruction>();
- tempInst.addAll(block2CellInst);
- try {
- this.executeInstructions(tempInst,ec);
- } catch (Exception e) {
-
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error executing "
- + tempInst.toString(), e);
- }
- }
-
- // now execute package function
- for (int i = 0; i < _inst.size(); i++)
- {
- try {
- if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
- executeInstruction(ec, (ExternalFunctionInvocationInstruction) _inst.get(i));
- }
- catch(Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() +
- "Failed to execute instruction " + _inst.get(i).toString(), e);
- }
- }
-
- // convert cell to block
- if( cell2BlockInst != null )
- {
- ArrayList<Instruction> tempInst = new ArrayList<Instruction>();
- try {
- tempInst.clear();
- tempInst.addAll(cell2BlockInst);
- this.executeInstructions(tempInst, ec);
- } catch (Exception e) {
-
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction "
- + cell2BlockInst.toString(), e);
- }
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- * Given a list of parameters as data identifiers, returns a string
- * representation.
- *
- * @param params
- * @return
- */
-
- protected String getParameterString(ArrayList<DataIdentifier> params) {
- String parameterString = "";
-
- for (int i = 0; i < params.size(); i++) {
- if (i != 0)
- parameterString += ",";
-
- DataIdentifier param = params.get(i);
-
- if (param.getDataType() == DataType.MATRIX) {
- String s = getDataTypeString(DataType.MATRIX) + ":";
- s = s + "" + param.getName() + "" + ":";
- s = s + getValueTypeString(param.getValueType());
- parameterString += s;
- continue;
- }
-
- if (param.getDataType() == DataType.SCALAR) {
- String s = getDataTypeString(DataType.SCALAR) + ":";
- s = s + "" + param.getName() + "" + ":";
- s = s + getValueTypeString(param.getValueType());
- parameterString += s;
- continue;
- }
-
- if (param.getDataType() == DataType.OBJECT) {
- String s = getDataTypeString(DataType.OBJECT) + ":";
- s = s + "" + param.getName() + "" + ":";
- parameterString += s;
- continue;
- }
- }
-
- return parameterString;
- }
-
- /**
- * method to get instructions
- */
- protected void createInstructions() {
-
- _inst = new ArrayList<Instruction>();
-
- // unblock all input matrices
- block2CellInst = getBlock2CellInstructions(getInputParams(),_unblockedFileNames);
-
- // assemble information provided through keyvalue pairs
- String className = _otherParams.get(ExternalFunctionStatement.CLASS_NAME);
- String configFile = _otherParams.get(ExternalFunctionStatement.CONFIG_FILE);
-
- // class name cannot be null, however, configFile and execLocation can
- // be null
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
-
- // assemble input and output param strings
- String inputParameterString = getParameterString(getInputParams());
- String outputParameterString = getParameterString(getOutputParams());
-
- // generate instruction
- ExternalFunctionInvocationInstruction einst = new ExternalFunctionInvocationInstruction(
- className, configFile, inputParameterString,
- outputParameterString);
-
- if (getInputParams().size() > 0)
- einst.setLocation(getInputParams().get(0));
- else if (getOutputParams().size() > 0)
- einst.setLocation(getOutputParams().get(0));
- else
- einst.setLocation(this._beginLine, this._endLine, this._beginColumn, this._endColumn);
-
- _inst.add(einst);
-
- // block output matrices
- cell2BlockInst = getCell2BlockInstructions(getOutputParams(),_blockedFileNames);
- }
-
-
- /**
- * Method to generate a reblock job to convert the cell representation into block representation
- * @param outputParams
- * @param blockedFileNames
- * @return
- */
- private ArrayList<Instruction> getCell2BlockInstructions(
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> blockedFileNames) {
-
- ArrayList<Instruction> c2binst = null;
-
- //list of matrices that need to be reblocked
- ArrayList<DataIdentifier> matrices = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<DataIdentifier>();
-
- // identify outputs that are matrices
- for (int i = 0; i < outputParams.size(); i++) {
- if (outputParams.get(i).getDataType() == DataType.MATRIX) {
- if( _skipOutReblock.contains(outputParams.get(i).getName()) )
- matricesNoReblock.add(outputParams.get(i));
- else
- matrices.add(outputParams.get(i));
- }
- }
-
- if( !matrices.isEmpty() )
- {
- c2binst = new ArrayList<Instruction>();
- MRJobInstruction reblkInst = new MRJobInstruction(JobType.REBLOCK);
- TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
- if(DMLScript.ENABLE_DEBUG_MODE) {
- MRJobLineNumbers = new TreeMap<Integer, ArrayList<String>>();
- }
-
- ArrayList<String> inLabels = new ArrayList<String>();
- ArrayList<String> outLabels = new ArrayList<String>();
- String[] outputs = new String[matrices.size()];
- byte[] resultIndex = new byte[matrices.size()];
- String reblock = "";
- String reblockStr = ""; //Keep a copy of a single MR reblock instruction
-
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
- try {
- // create a RBLK job that transforms each output matrix from cell to block
- for (int i = 0; i < matrices.size(); i++) {
- inLabels.add(matrices.get(i).getName());
- outLabels.add(matrices.get(i).getName() + "_extFnOutput");
- outputs[i] = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Output";
- blockedFileNames.put(matrices.get(i).getName(), outputs[i]);
- resultIndex[i] = (byte) i; // (matrices.size()+i);
-
- if (i > 0)
- reblock += Lop.INSTRUCTION_DELIMITOR;
-
- reblock += "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
-
- if(DMLScript.ENABLE_DEBUG_MODE) {
- //Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
- reblockStr = "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
- //Set MR reblock instruction line number (FOR DEBUGGER)
- if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
- MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
- }
- MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(reblockStr);
- }
- // create metadata instructions to populate symbol table
- // with variables that hold blocked matrices
-
- /*StringBuilder mtdInst = new StringBuilder();
- mtdInst.append("CP" + Lops.OPERAND_DELIMITOR + "createvar");
- mtdInst.append(Lops.OPERAND_DELIMITOR + outLabels.get(i) + Lops.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lops.VALUETYPE_PREFIX + matrices.get(i).getValueType());
- mtdInst.append(Lops.OPERAND_DELIMITOR + outputs[i] + Lops.DATATYPE_PREFIX + DataType.SCALAR + Lops.VALUETYPE_PREFIX + ValueType.STRING);
- mtdInst.append(Lops.OPERAND_DELIMITOR + OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo) ) ;
- c2binst.add(CPInstructionParser.parseSingleInstruction(mtdInst.toString()));*/
- Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo));
-
- createInst.setLocation(matrices.get(i));
-
- c2binst.add(createInst);
-
- }
-
- reblkInst.setReBlockInstructions(inLabels.toArray(new String[inLabels.size()]), "", reblock, "",
- outLabels.toArray(new String[inLabels.size()]), resultIndex, 1, 1);
- c2binst.add(reblkInst);
-
- // generate instructions that rename the output variables of REBLOCK job
- Instruction cpInst = null, rmInst = null;
- for (int i = 0; i < matrices.size(); i++) {
- cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
- rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
-
- cpInst.setLocation(matrices.get(i));
- rmInst.setLocation(matrices.get(i));
-
- c2binst.add(cpInst);
- c2binst.add(rmInst);
- //c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
- }
- } catch (Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
- }
-
- //LOGGING instructions
- if (LOG.isTraceEnabled()){
- LOG.trace("\n--- Cell-2-Block Instructions ---");
- for(Instruction i : c2binst) {
- LOG.trace(i.toString());
- }
- LOG.trace("----------------------------------");
- }
-
- }
-
- return c2binst; //null if no output matrices
- }
-
- /**
- * Method to generate instructions to convert input matrices from block to
- * cell. We generate a GMR job here.
- *
- * @param inputParams
- * @return
- */
- private ArrayList<Instruction> getBlock2CellInstructions(
- ArrayList<DataIdentifier> inputParams,
- HashMap<String, String> unBlockedFileNames) {
-
- ArrayList<Instruction> b2cinst = null;
-
- //list of input matrices
- ArrayList<DataIdentifier> matrices = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<DataIdentifier>();
-
- // find all inputs that are matrices
- for (int i = 0; i < inputParams.size(); i++) {
- if (inputParams.get(i).getDataType() == DataType.MATRIX) {
- if( _skipInReblock.contains(inputParams.get(i).getName()) )
- matricesNoReblock.add(inputParams.get(i));
- else
- matrices.add(inputParams.get(i));
- }
- }
-
- if( !matrices.isEmpty() )
- {
- b2cinst = new ArrayList<Instruction>();
- MRJobInstruction gmrInst = new MRJobInstruction(JobType.GMR);
- TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
- if(DMLScript.ENABLE_DEBUG_MODE) {
- MRJobLineNumbers = new TreeMap<Integer, ArrayList<String>>();
- }
- String gmrStr="";
- ArrayList<String> inLabels = new ArrayList<String>();
- ArrayList<String> outLabels = new ArrayList<String>();
- String[] outputs = new String[matrices.size()];
- byte[] resultIndex = new byte[matrices.size()];
-
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
-
- try {
- // create a GMR job that transforms each of these matrices from block to cell
- for (int i = 0; i < matrices.size(); i++) {
-
- //inputs[i] = "##" + matrices.get(i).getName() + "##";
- //inputInfo[i] = binBlockInputInfo;
- //outputInfo[i] = textCellOutputInfo;
- //numRows[i] = numCols[i] = numRowsPerBlock[i] = numColsPerBlock[i] = -1;
- //resultDimsUnknown[i] = 1;
-
- inLabels.add(matrices.get(i).getName());
- outLabels.add(matrices.get(i).getName()+"_extFnInput");
- resultIndex[i] = (byte) i; //(matrices.size()+i);
-
- outputs[i] = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
- unBlockedFileNames.put(matrices.get(i).getName(), outputs[i]);
-
- if(DMLScript.ENABLE_DEBUG_MODE) {
- //Create a dummy gmr instruction (FOR DEBUGGER)
- gmrStr = "MR" + Lop.OPERAND_DELIMITOR + "gmr" + Lop.OPERAND_DELIMITOR +
- i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + Lop.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize;
-
- //Set MR gmr instruction line number (FOR DEBUGGER)
- if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
- MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
- }
- MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(gmrStr);
- }
- // create metadata instructions to populate symbol table
- // with variables that hold unblocked matrices
-
- /*StringBuilder mtdInst = new StringBuilder();
- mtdInst.append("CP" + Lops.OPERAND_DELIMITOR + "createvar");
- mtdInst.append(Lops.OPERAND_DELIMITOR + outLabels.get(i) + Lops.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lops.VALUETYPE_PREFIX + matrices.get(i).getValueType());
- mtdInst.append(Lops.OPERAND_DELIMITOR + outputs[i] + Lops.DATATYPE_PREFIX + DataType.SCALAR + Lops.VALUETYPE_PREFIX + ValueType.STRING);
- mtdInst.append(Lops.OPERAND_DELIMITOR + OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo) ) ;
- b2cinst.add(CPInstructionParser.parseSingleInstruction(mtdInst.toString()));*/
-
- Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
-
- createInst.setLocation(matrices.get(i));
-
- b2cinst.add(createInst);
- }
-
- // Finally, generate GMR instruction that performs block2cell conversion
- gmrInst.setGMRInstructions(inLabels.toArray(new String[inLabels.size()]), "", "", "", "",
- outLabels.toArray(new String[outLabels.size()]), resultIndex, 0, 1);
-
- b2cinst.add(gmrInst);
-
- // generate instructions that rename the output variables of GMR job
- Instruction cpInst=null, rmInst=null;
- for (int i = 0; i < matrices.size(); i++) {
- cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
- rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
-
- cpInst.setLocation(matrices.get(i));
- rmInst.setLocation(matrices.get(i));
-
- b2cinst.add(cpInst);
- b2cinst.add(rmInst);
- }
- } catch (Exception e) {
- throw new PackageRuntimeException(e);
- }
-
- //LOG instructions
- if (LOG.isTraceEnabled()){
- LOG.trace("\n--- Block-2-Cell Instructions ---");
- for(Instruction i : b2cinst) {
- LOG.trace(i.toString());
- }
- LOG.trace("----------------------------------");
- }
- }
-
- //BEGIN FUNCTION PATCH
- if( !matricesNoReblock.isEmpty() )
- {
- //if( b2cinst==null )
- // b2cinst = new ArrayList<Instruction>();
-
- for( int i=0; i<matricesNoReblock.size(); i++ )
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
- try{
- String filename = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
- //String outLabel = matricesNoReblock.get(i).getName()+"_extFnInput";
- //Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabel, filename, false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
- //Instruction cpInst = VariableCPInstruction.prepareCopyInstruction( matricesNoReblock.get(i).getName(), outLabel);
-
- unBlockedFileNames.put(matricesNoReblock.get(i).getName(), filename); //
-
- //b2cinst.add(createInst);
- //b2cinst.add(cpInst);
- }
- catch (Exception e) {
- throw new PackageRuntimeException(e);
- }
-
- }
- }
- //END FUNCTION PATCH
-
- return b2cinst; //null if no input matrices
- }
-
- /**
- * Method to execute an external function invocation instruction.
- *
- * @param inst
- * @param dQueue
- * @throws NimbleCheckedRuntimeException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("unchecked")
- public void executeInstruction(ExecutionContext ec, ExternalFunctionInvocationInstruction inst)
- throws DMLRuntimeException
- {
- String className = inst.getClassName();
- String configFile = inst.getConfigFile();
-
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class name can't be null");
-
- // create instance of package function.
- Object o;
- try
- {
- Class<Instruction> cla = (Class<Instruction>) Class.forName(className);
- o = cla.newInstance();
- }
- catch (Exception e)
- {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error generating package function object " ,e );
- }
-
- if (!(o instanceof PackageFunction))
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class is not of type PackageFunction");
-
- PackageFunction func = (PackageFunction) o;
-
- // add inputs to this package function based on input parameter
- // and their mappings.
- setupInputs(func, inst.getInputParams(), ec.getVariables());
- func.setConfiguration(configFile);
- func.setBaseDir(_baseDir);
-
- //executes function
- func.execute();
-
- // verify output of function execution matches declaration
- // and add outputs to variableMapping and Metadata
- verifyAndAttachOutputs(ec, func, inst.getOutputParams());
- }
-
- /**
- * Method to verify that function outputs match with declared outputs
- *
- * @param returnFunc
- * @param outputParams
- * @throws DMLRuntimeException
- */
- protected void verifyAndAttachOutputs(ExecutionContext ec, PackageFunction returnFunc,
- String outputParams) throws DMLRuntimeException {
-
- ArrayList<String> outputs = getParameters(outputParams);
- // make sure they are of equal size first
-
- if (outputs.size() != returnFunc.getNumFunctionOutputs()) {
- throw new PackageRuntimeException(
- "Number of function outputs ("+returnFunc.getNumFunctionOutputs()+") " +
- "does not match with declaration ("+outputs.size()+").");
- }
-
- // iterate over each output and verify that type matches
- for (int i = 0; i < outputs.size(); i++) {
- StringTokenizer tk = new StringTokenizer(outputs.get(i), ":");
- ArrayList<String> tokens = new ArrayList<String>();
- while (tk.hasMoreTokens()) {
- tokens.add(tk.nextToken());
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Matrix) {
- Matrix m = (Matrix) returnFunc.getFunctionOutput(i);
-
- if (!(tokens.get(0)
- .compareTo(getFunctionParameterDataTypeString(FunctionParameterType.Matrix)) == 0)
- || !(tokens.get(2).compareTo(
- getMatrixValueTypeString(m.getValueType())) == 0)) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- // add result to variableMapping
- String varName = tokens.get(1);
- MatrixObject newVar = createOutputMatrixObject( m );
- newVar.setVarName(varName);
-
- /* cleanup not required because done at central position (FunctionCallCPInstruction)
- MatrixObjectNew oldVar = (MatrixObjectNew)getVariable(varName);
- if( oldVar!=null )
- oldVar.clearData();*/
-
- //getVariables().put(varName, newVar); //put/override in local symbol table
- ec.setVariable(varName, newVar);
-
- continue;
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Scalar) {
- Scalar s = (Scalar) returnFunc.getFunctionOutput(i);
-
- if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Scalar))
- || !tokens.get(2).equals(
- getScalarValueTypeString(s.getScalarType()))) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- // allocate and set appropriate object based on type
- ScalarObject scalarObject = null;
- ScalarValueType type = s.getScalarType();
- switch (type) {
- case Integer:
- scalarObject = new IntObject(tokens.get(1),
- Long.parseLong(s.getValue()));
- break;
- case Double:
- scalarObject = new DoubleObject(tokens.get(1),
- Double.parseDouble(s.getValue()));
- break;
- case Boolean:
- scalarObject = new BooleanObject(tokens.get(1),
- Boolean.parseBoolean(s.getValue()));
- break;
- case Text:
- scalarObject = new StringObject(tokens.get(1), s.getValue());
- break;
- default:
- throw new PackageRuntimeException(
- "Unknown scalar value type '"+type+"' of output '"+outputs.get(i)+"'.");
- }
-
- //this.getVariables().put(tokens.get(1), scalarObject);
- ec.setVariable(tokens.get(1), scalarObject);
- continue;
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Object) {
- if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Object))) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- throw new PackageRuntimeException(
- "Object types not yet supported");
-
- // continue;
- }
-
- throw new PackageRuntimeException(
- "Unknown data type '"+returnFunc.getFunctionOutput(i).getType()+"' " +
- "of output '"+outputs.get(i)+"'.");
- }
- }
-
- protected MatrixObject createOutputMatrixObject( Matrix m )
- throws CacheException
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
- return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
- }
-
- /**
- * Method to get string representation of scalar value type
- *
- * @param scalarType
- * @return
- */
-
- protected String getScalarValueTypeString(ScalarValueType scalarType) {
-
- if (scalarType.equals(ScalarValueType.Double))
- return "Double";
- if (scalarType.equals(ScalarValueType.Integer))
- return "Integer";
- if (scalarType.equals(ScalarValueType.Boolean))
- return "Boolean";
- if (scalarType.equals(ScalarValueType.Text))
- return "String";
-
- throw new PackageRuntimeException("Unknown scalar value type");
- }
-
- /**
- * Method to parse inputs, update labels, and add to package function.
- *
- * @param func
- * @param inputParams
- * @param metaData
- * @param variableMapping
- */
- protected void setupInputs (PackageFunction func, String inputParams,
- LocalVariableMap variableMapping) {
-
- ArrayList<String> inputs = getParameters(inputParams);
- ArrayList<FunctionParameter> inputObjects = getInputObjects(inputs, variableMapping);
- func.setNumFunctionInputs(inputObjects.size());
- for (int i = 0; i < inputObjects.size(); i++)
- func.setInput(inputObjects.get(i), i);
-
- }
-
- /**
- * Method to convert string representation of input into function input
- * object.
- *
- * @param inputs
- * @param variableMapping
- * @param metaData
- * @return
- */
-
- protected ArrayList<FunctionParameter> getInputObjects(ArrayList<String> inputs,
- LocalVariableMap variableMapping) {
- ArrayList<FunctionParameter> inputObjects = new ArrayList<FunctionParameter>();
-
- for (int i = 0; i < inputs.size(); i++) {
- ArrayList<String> tokens = new ArrayList<String>();
- StringTokenizer tk = new StringTokenizer(inputs.get(i), ":");
- while (tk.hasMoreTokens()) {
- tokens.add(tk.nextToken());
- }
-
- if (tokens.get(0).equals("Matrix")) {
- String varName = tokens.get(1);
- MatrixObject mobj = (MatrixObject) variableMapping.get(varName);
- MatrixCharacteristics mc = mobj.getMatrixCharacteristics();
- Matrix m = new Matrix(mobj.getFileName(),
- mc.getRows(), mc.getCols(),
- getMatrixValueType(tokens.get(2)));
- modifyInputMatrix(m, mobj);
- inputObjects.add(m);
- }
-
- if (tokens.get(0).equals("Scalar")) {
- String varName = tokens.get(1);
- ScalarObject so = (ScalarObject) variableMapping.get(varName);
- Scalar s = new Scalar(getScalarValueType(tokens.get(2)),
- so.getStringValue());
- inputObjects.add(s);
-
- }
-
- if (tokens.get(0).equals("Object")) {
- String varName = tokens.get(1);
- Object o = variableMapping.get(varName);
- BinaryObject obj = new BinaryObject(o);
- inputObjects.add(obj);
-
- }
- }
-
- return inputObjects;
-
- }
-
- protected void modifyInputMatrix(Matrix m, MatrixObject mobj)
- {
- //do nothing, intended for extensions
- }
-
- /**
- * Converts string representation of scalar value type to enum type
- *
- * @param string
- * @return
- */
- protected ScalarValueType getScalarValueType(String string) {
- if (string.equals("Double"))
- return ScalarValueType.Double;
- if (string.equals("Integer"))
- return ScalarValueType.Integer;
- if (string.equals("Boolean"))
- return ScalarValueType.Boolean;
- if (string.equals("String"))
- return ScalarValueType.Text;
-
- throw new PackageRuntimeException("Unknown scalar type");
-
- }
-
- /**
- * Get string representation of matrix value type
- *
- * @param t
- * @return
- */
-
- protected String getMatrixValueTypeString(Matrix.ValueType t) {
- if (t.equals(Matrix.ValueType.Double))
- return "Double";
-
- if (t.equals(Matrix.ValueType.Integer))
- return "Integer";
-
- throw new PackageRuntimeException("Unknown matrix value type");
- }
-
- /**
- * Converts string representation of matrix value type into enum type
- *
- * @param string
- * @return
- */
-
- protected com.ibm.bi.dml.udf.Matrix.ValueType getMatrixValueType(String string) {
-
- if (string.equals("Double"))
- return Matrix.ValueType.Double;
- if (string.equals("Integer"))
- return Matrix.ValueType.Integer;
-
- throw new PackageRuntimeException("Unknown matrix value type");
-
- }
-
- /**
- * Method to break the comma separated input parameters into an arraylist of
- * parameters
- *
- * @param inputParams
- * @return
- */
- protected ArrayList<String> getParameters(String inputParams) {
- ArrayList<String> inputs = new ArrayList<String>();
-
- StringTokenizer tk = new StringTokenizer(inputParams, ",");
- while (tk.hasMoreTokens()) {
- inputs.add(tk.nextToken());
- }
-
- return inputs;
- }
-
- /**
- * Get string representation for data type
- *
- * @param d
- * @return
- */
- protected String getDataTypeString(DataType d) {
- if (d.equals(DataType.MATRIX))
- return "Matrix";
-
- if (d.equals(DataType.SCALAR))
- return "Scalar";
-
- if (d.equals(DataType.OBJECT))
- return "Object";
-
- throw new PackageRuntimeException("Should never come here");
-
- }
-
- /**
- * Method to get string representation of data type.
- *
- * @param t
- * @return
- */
- protected String getFunctionParameterDataTypeString(FunctionParameterType t) {
- if (t.equals(FunctionParameterType.Matrix))
- return "Matrix";
-
- if (t.equals(FunctionParameterType.Scalar))
- return "Scalar";
-
- if (t.equals(FunctionParameterType.Object))
- return "Object";
-
- throw new PackageRuntimeException("Should never come here");
- }
-
- /**
- * Get string representation of value type
- *
- * @param v
- * @return
- */
- protected String getValueTypeString(ValueType v) {
- if (v.equals(ValueType.DOUBLE))
- return "Double";
-
- if (v.equals(ValueType.INT))
- return "Integer";
-
- if (v.equals(ValueType.BOOLEAN))
- return "Boolean";
-
- if (v.equals(ValueType.STRING))
- return "String";
-
- throw new PackageRuntimeException("Should never come here");
- }
-
- public void printMe() {
- //System.out.println("***** INSTRUCTION BLOCK *****");
- for (Instruction i : this._inst) {
- i.printMe();
- }
- }
-
- public HashMap<String,String> getOtherParams()
- {
- return _otherParams;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in external function program block generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-
- /////////////////////////////////////////////////
- // Extension for Global Data Flow Optimization
- // by Mathias Peters
- ///////
-
- //FUNCTION PATCH
-
- private Collection<String> _skipInReblock = new HashSet<String>();
- private Collection<String> _skipOutReblock = new HashSet<String>();
-
- public void setSkippedReblockLists( Collection<String> varsIn, Collection<String> varsOut )
- {
- _skipInReblock.clear();
- _skipOutReblock.clear();
-
- if( varsIn!=null || varsOut!=null )
- {
- if( varsIn != null )
- _skipInReblock.addAll(varsIn);
- if( varsOut != null )
- _skipOutReblock.addAll(varsOut);
-
- //regenerate instructions
- createInstructions();
- }
- }
-
-
- @Override
- public ArrayList<Instruction> getInstructions()
- {
- ArrayList<Instruction> tmp = new ArrayList<Instruction>();
- if( cell2BlockInst != null )
- tmp.addAll(cell2BlockInst);
- if( block2CellInst != null )
- tmp.addAll(block2CellInst);
- return tmp;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
deleted file mode 100644
index ebbd53d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-
-/**
- * CP external function program block, that overcomes the need for
- * BlockToCell and CellToBlock MR jobs by changing the contract for an external function.
- * If execlocation="CP", the implementation of an external function must read and write
- * matrices as InputInfo.BinaryBlockInputInfo and OutputInfo.BinaryBlockOutputInfo.
- *
- * Furthermore, it extends ExternalFunctionProgramBlock with a base directory in order
- * to make it parallelizable, even in case of different JVMs. For this purpose every
- * external function must implement a <SET_BASE_DIR> method.
- *
- *
- */
-public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
-{
-
- public static String DEFAULT_FILENAME = "ext_funct";
- private static IDSequence _defaultSeq = new IDSequence();
-
- /**
- * Constructor that also provides otherParams that are needed for external
- * functions. Remaining parameters will just be passed to constructor for
- * function program block.
- *
- * @param eFuncStat
- * @throws DMLRuntimeException
- */
- public ExternalFunctionProgramBlockCP(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> otherParams,
- String baseDir) throws DMLRuntimeException {
-
- super(prog, inputParams, outputParams, baseDir); //w/o instruction generation
-
- // copy other params
- _otherParams = new HashMap<String, String>();
- _otherParams.putAll(otherParams);
-
- // generate instructions (overwritten)
- createInstructions();
- }
-
- /**
- * Method to be invoked to execute instructions for the external function
- * invocation
- * @throws DMLRuntimeException
- */
- @Override
- public void execute(ExecutionContext ec) throws DMLRuntimeException
- {
- _runID = _idSeq.getNextID();
-
- ExternalFunctionInvocationInstruction inst = null;
-
- // execute package function
- for (int i=0; i < _inst.size(); i++)
- {
- try {
- inst = (ExternalFunctionInvocationInstruction)_inst.get(i);
- executeInstruction( ec, inst );
- }
- catch (Exception e){
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating instruction " + i + " in external function programBlock. inst: " + inst.toString(), e);
- }
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- * Executes the external function instruction without the use of NIMBLE tasks.
- *
- * @param inst
- * @throws DMLRuntimeException
- * @throws NimbleCheckedRuntimeException
- */
- @Override
- public void executeInstruction(ExecutionContext ec, ExternalFunctionInvocationInstruction inst)
- throws DMLRuntimeException
- {
- // After removal of nimble, we moved the code of ExternalFunctionProgramBlockCP to
- // ExternalFunctionProgramBlock and hence hence both types of external functions can
- // share the same code path here.
- super.executeInstruction(ec, inst);
- }
-
-
- @Override
- protected void createInstructions()
- {
- _inst = new ArrayList<Instruction>();
-
- // assemble information provided through keyvalue pairs
- String className = _otherParams.get(ExternalFunctionStatement.CLASS_NAME);
- String configFile = _otherParams.get(ExternalFunctionStatement.CONFIG_FILE);
-
- // class name cannot be null, however, configFile and execLocation can be null
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
-
- // assemble input and output param strings
- String inputParameterString = getParameterString(getInputParams());
- String outputParameterString = getParameterString(getOutputParams());
-
- // generate instruction
- ExternalFunctionInvocationInstruction einst = new ExternalFunctionInvocationInstruction(
- className, configFile, inputParameterString,
- outputParameterString);
-
- _inst.add(einst);
-
- }
-
- @Override
- protected void modifyInputMatrix(Matrix m, MatrixObject mobj)
- {
- //pass in-memory object to external function
- m.setMatrixObject( mobj );
- }
-
- @Override
- protected MatrixObject createOutputMatrixObject(Matrix m)
- {
- MatrixObject ret = m.getMatrixObject();
-
- if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
- }
-
- //for allowing in-memory packagesupport matrices w/o filesnames
- if( ret.getFileName().equals( DEFAULT_FILENAME ) )
- {
- ret.setFileName( createDefaultOutputFilePathAndName() );
- }
-
- return ret;
- }
-
-
- public String createDefaultOutputFilePathAndName( )
- {
- return _baseDir + DEFAULT_FILENAME + _defaultSeq.getNextID();
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in external function program block (for CP) generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
deleted file mode 100644
index c40420c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-public class ForProgramBlock extends ProgramBlock
-{
-
- protected ArrayList<Instruction> _fromInstructions;
- protected ArrayList<Instruction> _toInstructions;
- protected ArrayList<Instruction> _incrementInstructions;
-
- protected ArrayList <Instruction> _exitInstructions ;
- protected ArrayList<ProgramBlock> _childBlocks;
-
- protected String[] _iterablePredicateVars; //from,to,where constants/internal vars not captured via instructions
-
- public void printMe()
- {
- LOG.debug("***** current for block predicate inst: *****");
- LOG.debug("FROM:");
- for (Instruction cp : _fromInstructions){
- cp.printMe();
- }
- LOG.debug("TO:");
- for (Instruction cp : _toInstructions){
- cp.printMe();
- }
- LOG.debug("INCREMENT:");
- for (Instruction cp : _incrementInstructions){
- cp.printMe();
- }
-
- LOG.debug("***** children block inst: *****");
- for (ProgramBlock pb : this._childBlocks){
- pb.printMe();
- }
-
- LOG.debug("***** current block inst exit: *****");
- for (Instruction i : this._exitInstructions) {
- i.printMe();
- }
-
- }
-
-
- public ForProgramBlock(Program prog, String[] iterPredVars) throws DMLRuntimeException
- {
- super(prog);
-
- _exitInstructions = new ArrayList<Instruction>();
- _childBlocks = new ArrayList<ProgramBlock>();
- _iterablePredicateVars = iterPredVars;
- }
-
- public ArrayList<Instruction> getFromInstructions()
- {
- return _fromInstructions;
- }
-
- public void setFromInstructions(ArrayList<Instruction> instructions)
- {
- _fromInstructions = instructions;
- }
-
- public ArrayList<Instruction> getToInstructions()
- {
- return _toInstructions;
- }
-
- public void setToInstructions(ArrayList<Instruction> instructions)
- {
- _toInstructions = instructions;
- }
-
- public ArrayList<Instruction> getIncrementInstructions()
- {
- return _incrementInstructions;
- }
-
- public void setIncrementInstructions(ArrayList<Instruction> instructions)
- {
- _incrementInstructions = instructions;
- }
-
- public void addExitInstruction(Instruction inst){
- _exitInstructions.add(inst);
- }
-
- public ArrayList<Instruction> getExitInstructions(){
- return _exitInstructions;
- }
-
- public void setExitInstructions(ArrayList<Instruction> inst){
- _exitInstructions = inst;
- }
-
-
- public void addProgramBlock(ProgramBlock childBlock) {
- _childBlocks.add(childBlock);
- }
-
- public ArrayList<ProgramBlock> getChildBlocks()
- {
- return _childBlocks;
- }
-
- public void setChildBlocks(ArrayList<ProgramBlock> pbs)
- {
- _childBlocks = pbs;
- }
-
- public String[] getIterablePredicateVars()
- {
- return _iterablePredicateVars;
- }
-
- public void setIterablePredicateVars(String[] iterPredVars)
- {
- _iterablePredicateVars = iterPredVars;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- // add the iterable predicate variable to the variable set
- String iterVarName = _iterablePredicateVars[0];
-
- // evaluate from, to, incr only once (assumption: known at for entry)
- IntObject from = executePredicateInstructions( 1, _fromInstructions, ec );
- IntObject to = executePredicateInstructions( 2, _toInstructions, ec );
- IntObject incr = executePredicateInstructions( 3, _incrementInstructions, ec );
-
- if ( incr.getLongValue() <= 0 ) //would produce infinite loop
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment of variable '" + iterVarName + "' must evaluate to a positive value.");
-
- // initialize iter var to from value
- IntObject iterVar = new IntObject(iterVarName, from.getLongValue() );
-
- // execute for loop
- try
- {
- // run for loop body as long as predicate is true
- // (for supporting dynamic TO, move expression execution to end of while loop)
- while( iterVar.getLongValue() <= to.getLongValue() )
- {
- ec.setVariable(iterVarName, iterVar);
-
- //for all child blocks
- for (int i=0 ; i < this._childBlocks.size() ; i++) {
- ec.updateDebugState( i );
- _childBlocks.get(i).execute(ec);
- }
-
- // update the iterable predicate variable
- if(ec.getVariable(iterVarName) == null || !(ec.getVariable(iterVarName) instanceof IntObject))
- throw new DMLRuntimeException("Iterable predicate variable " + iterVarName + " must remain of type scalar int.");
-
- //increment of iterVar (changes in loop body get discarded)
- iterVar = new IntObject( iterVarName, iterVar.getLongValue()+incr.getLongValue() );
- }
- }
- catch (DMLScriptException e)
- {
- throw e;
- }
- catch (Exception e)
- {
- throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating for program block", e);
- }
-
- //execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch (Exception e){
- throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating for exit instructions", e);
- }
- }
-
- /**
- *
- * @param pos
- * @param instructions
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- protected IntObject executePredicateInstructions( int pos, ArrayList<Instruction> instructions, ExecutionContext ec )
- throws DMLRuntimeException
- {
- ScalarObject tmp = null;
- IntObject ret = null;
-
- try
- {
- if( _iterablePredicateVars[pos] != null )
- {
- //probe for scalar variables
- Data ldat = ec.getVariable( _iterablePredicateVars[pos] );
- if( ldat != null && ldat instanceof ScalarObject )
- tmp = (ScalarObject)ldat;
- else //handle literals
- tmp = new IntObject( UtilFunctions.parseToLong(_iterablePredicateVars[pos]) );
- }
- else
- {
- if( _sb!=null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- ForStatementBlock fsb = (ForStatementBlock)_sb;
- Hop predHops = null;
- boolean recompile = false;
- if (pos == 1){
- predHops = fsb.getFromHops();
- recompile = fsb.requiresFromRecompilation();
- }
- else if (pos == 2) {
- predHops = fsb.getToHops();
- recompile = fsb.requiresToRecompilation();
- }
- else if (pos == 3){
- predHops = fsb.getIncrementHops();
- recompile = fsb.requiresIncrementRecompilation();
- }
- tmp = (IntObject) executePredicate(instructions, predHops, recompile, ValueType.INT, ec);
- }
- else
- tmp = (IntObject) executePredicate(instructions, null, false, ValueType.INT, ec);
- }
- }
- catch(Exception ex)
- {
- String predNameStr = null;
- if (pos == 1) predNameStr = "from";
- else if (pos == 2) predNameStr = "to";
- else if (pos == 3) predNameStr = "increment";
-
- throw new DMLRuntimeException(this.printBlockErrorLocation() +"Error evaluating '" + predNameStr + "' predicate", ex);
- }
-
- //final check of resulting int object (guaranteed to be non-null, see executePredicate)
- if( tmp instanceof IntObject )
- ret = (IntObject)tmp;
- else //downcast to int if necessary
- ret = new IntObject(tmp.getName(),tmp.getLongValue());
-
- return ret;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in for program block generated from for statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
deleted file mode 100644
index 9785a38..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.utils.Statistics;
-
-
-public class FunctionProgramBlock extends ProgramBlock
-{
-
- protected ArrayList<ProgramBlock> _childBlocks;
- protected ArrayList<DataIdentifier> _inputParams;
- protected ArrayList<DataIdentifier> _outputParams;
-
- private boolean _recompileOnce = false;
-
- public FunctionProgramBlock( Program prog, ArrayList<DataIdentifier> inputParams, ArrayList<DataIdentifier> outputParams) throws DMLRuntimeException
- {
- super(prog);
- _childBlocks = new ArrayList<ProgramBlock>();
- _inputParams = new ArrayList<DataIdentifier>();
- for (DataIdentifier id : inputParams){
- _inputParams.add(new DataIdentifier(id));
-
- }
- _outputParams = new ArrayList<DataIdentifier>();
- for (DataIdentifier id : outputParams){
- _outputParams.add(new DataIdentifier(id));
- }
- }
-
- public ArrayList<DataIdentifier> getInputParams(){
- return _inputParams;
- }
-
- public ArrayList<DataIdentifier> getOutputParams(){
- return _outputParams;
- }
-
- public void addProgramBlock(ProgramBlock childBlock) {
- _childBlocks.add(childBlock);
- }
-
- public void setChildBlocks( ArrayList<ProgramBlock> pbs)
- {
- _childBlocks = pbs;
- }
-
- public ArrayList<ProgramBlock> getChildBlocks() {
- return _childBlocks;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //dynamically recompile entire function body (according to function inputs)
- try {
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && isRecompileOnce()
- && ParForProgramBlock.RESET_RECOMPILATION_FLAGs )
- {
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- //note: it is important to reset the recompilation flags here
- // (1) it is safe to reset recompilation flags because a 'recompile_once'
- // function will be recompiled for every execution.
- // (2) without reset, there would be no benefit in recompiling the entire function
- LocalVariableMap tmp = (LocalVariableMap) ec.getVariables().clone();
- Recompiler.recompileProgramBlockHierarchy(_childBlocks, tmp, _tid, true);
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.incrementFunRecompileTime(t1-t0);
- Statistics.incrementFunRecompiles();
- }
- }
- }
- catch(Exception ex) {
- throw new DMLRuntimeException("Error recompiling function body.", ex);
- }
-
- // for each program block
- try {
- for (int i=0 ; i < this._childBlocks.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocks.get(i).execute(ec);
- }
- }
- catch (DMLScriptException e) {
- throw e;
- }
- catch (Exception e){
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating function program block", e);
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- *
- * @param vars
- */
- protected void checkOutputParameters( LocalVariableMap vars )
- {
- for( DataIdentifier diOut : _outputParams )
- {
- String varName = diOut.getName();
- Data dat = vars.get( varName );
- if( dat == null )
- LOG.error("Function output "+ varName +" is missing.");
- else if( dat.getDataType() != diOut.getDataType() )
- LOG.warn("Function output "+ varName +" has wrong data type: "+dat.getDataType()+".");
- else if( dat.getValueType() != diOut.getValueType() )
- LOG.warn("Function output "+ varName +" has wrong value type: "+dat.getValueType()+".");
-
- }
- }
-
- public void setRecompileOnce( boolean flag ) {
- _recompileOnce = flag;
- }
-
- public boolean isRecompileOnce() {
- return _recompileOnce;
- }
-
- public void printMe() {
-
- for (ProgramBlock pb : this._childBlocks){
- pb.printMe();
- }
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in function program block generated from function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
deleted file mode 100644
index 925b426..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-
-public class IfProgramBlock extends ProgramBlock
-{
-
- private ArrayList<Instruction> _predicate;
- private String _predicateResultVar;
- private ArrayList <Instruction> _exitInstructions ;
- private ArrayList<ProgramBlock> _childBlocksIfBody;
- private ArrayList<ProgramBlock> _childBlocksElseBody;
-
- public IfProgramBlock(Program prog, ArrayList<Instruction> predicate) throws DMLRuntimeException{
- super(prog);
-
- _childBlocksIfBody = new ArrayList<ProgramBlock>();
- _childBlocksElseBody = new ArrayList<ProgramBlock>();
-
- _predicate = predicate;
- _predicateResultVar = findPredicateResultVar ();
- _exitInstructions = new ArrayList<Instruction>();
- }
-
- public ArrayList<ProgramBlock> getChildBlocksIfBody()
- { return _childBlocksIfBody; }
-
- public void setChildBlocksIfBody(ArrayList<ProgramBlock> blocks)
- { _childBlocksIfBody = blocks; }
-
- public void addProgramBlockIfBody(ProgramBlock pb)
- { _childBlocksIfBody.add(pb); }
-
- public ArrayList<ProgramBlock> getChildBlocksElseBody()
- { return _childBlocksElseBody; }
-
- public void setChildBlocksElseBody(ArrayList<ProgramBlock> blocks)
- { _childBlocksElseBody = blocks; }
-
- public void addProgramBlockElseBody(ProgramBlock pb)
- { _childBlocksElseBody.add(pb); }
-
- public void printMe() {
-
- LOG.debug("***** if current block predicate inst: *****");
- for (Instruction cp : _predicate){
- cp.printMe();
- }
-
- LOG.debug("***** children block inst --- if body : *****");
- for (ProgramBlock pb : this._childBlocksIfBody){
- pb.printMe();
- }
-
- LOG.debug("***** children block inst --- else body : *****");
- for (ProgramBlock pb: this._childBlocksElseBody){
- pb.printMe();
- }
-
- LOG.debug("***** current block inst exit: *****");
- for (Instruction i : this._exitInstructions) {
- i.printMe();
- }
- }
-
-
- public void setExitInstructions2(ArrayList<Instruction> exitInstructions){
- _exitInstructions = exitInstructions;
- }
-
- public void setExitInstructions1(ArrayList<Instruction> predicate){
- _predicate = predicate;
- }
-
- public void addExitInstruction(Instruction inst){
- _exitInstructions.add(inst);
- }
-
- public ArrayList<Instruction> getPredicate(){
- return _predicate;
- }
-
- public void setPredicate(ArrayList<Instruction> predicate)
- {
- _predicate = predicate;
-
- //update result var if non-empty predicate (otherwise,
- //do not overwrite varname predicate in predicateResultVar)
- if( _predicate != null && !_predicate.isEmpty() )
- _predicateResultVar = findPredicateResultVar();
- }
-
- public String getPredicateResultVar(){
- return _predicateResultVar;
- }
-
- public void setPredicateResultVar(String resultVar) {
- _predicateResultVar = resultVar;
- }
-
- public ArrayList<Instruction> getExitInstructions(){
- return _exitInstructions;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- BooleanObject predResult = executePredicate(ec);
-
- //execute if statement
- if(predResult.getBooleanValue())
- {
- try
- {
- for (int i=0 ; i < _childBlocksIfBody.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocksIfBody.get(i).execute(ec);
- }
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch(Exception e)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating if statement body ", e);
- }
- }
- else
- {
- try
- {
- for (int i=0 ; i < _childBlocksElseBody.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocksElseBody.get(i).execute(ec);
- }
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch(Exception e)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating else statement body ", e);
- }
- }
-
- //execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch (Exception e){
-
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating if exit instructions ", e);
- }
- }
-
- /**
- *
- * @param ec
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private BooleanObject executePredicate(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- BooleanObject result = null;
- try
- {
- if( _predicate!=null && !_predicate.isEmpty() )
- {
- if( _sb != null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- IfStatementBlock isb = (IfStatementBlock)_sb;
- Hop predicateOp = isb.getPredicateHops();
- boolean recompile = isb.requiresPredicateRecompilation();
- result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
- }
- else
- result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
- }
- else
- {
- //get result var
- ScalarObject scalarResult = null;
- Data resultData = ec.getVariable(_predicateResultVar);
- if ( resultData == null ) {
- //note: resultvar is a literal (can it be of any value type other than String, hence no literal/varname conflict)
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, true);
- }
- else {
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, false);
- }
-
- //check for invalid type String
- if (scalarResult instanceof StringObject)
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "\nIf predicate variable "+ _predicateResultVar + " evaluated to string " + scalarResult + " which is not allowed for predicates in DML");
-
- //process result
- if( scalarResult instanceof BooleanObject )
- result = (BooleanObject)scalarResult;
- else
- result = new BooleanObject( scalarResult.getBooleanValue() ); //auto casting
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to evaluate the IF predicate.", ex);
- }
-
- //(guaranteed to be non-null, see executePredicate/getScalarInput)
- return result;
- }
-
- private String findPredicateResultVar ( ) {
- String result = null;
- for ( Instruction si : _predicate ) {
- if ( si.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM && ((CPInstruction)si).getCPInstructionType() != CPINSTRUCTION_TYPE.Variable ) {
- result = ((ComputationCPInstruction) si).getOutputVariableName();
- }
- else if(si instanceof VariableCPInstruction && ((VariableCPInstruction)si).isVariableCastInstruction()){
- result = ((VariableCPInstruction)si).getOutputVariableName();
- }
- }
- return result;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in if program block generated from if statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
deleted file mode 100644
index 5c5132d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LineageObject;
-
-import java.util.HashMap;
-import java.util.StringTokenizer;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/**
- * Replaces <code>HashMap⟨String, Data⟩</code> as the table of
- * variable names and references. No longer supports global consistency.
- *
- */
-public class LocalVariableMap implements Cloneable
-{
- private static String eol = System.getProperty ("line.separator");
- private static String ELEMENT_DELIM = com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter.ELEMENT_DELIM;
- private static IDSequence _seq = new IDSequence();
-
- private HashMap <String, Data> localMap = null;
- private final long localID;
-
- public LocalVariableMap()
- {
- localMap = new HashMap <String, Data>();
- localID = _seq.getNextID();
- }
-
- public LocalVariableMap(LocalVariableMap vars)
- {
- localMap = new HashMap <String, Data>(vars.localMap);
- localID = _seq.getNextID();
- }
-
- /**
- *
- * @return
- */
- public Set<String> keySet()
- {
- return localMap.keySet();
- }
-
- /**
- * Retrieves the data object given its name.
- *
- * @param name : the variable name for the data object
- * @return the direct reference to the data object
- */
- public Data get( String name )
- {
- return localMap.get( name );
- }
-
- /**
- * Adds a new (name, value) pair to the variable map, or replaces an old pair with
- * the same name. Several different variable names may refer to the same value.
- *
- * @param name : the variable name for the data value
- * @param val : the data value object (such as envelope)
- */
- public void put(String name, Data val)
- {
- localMap.put( name, val );
- }
-
- /**
- *
- * @param vars
- */
- public void putAll( LocalVariableMap vars )
- {
- if( vars == this || vars == null )
- return;
- localMap.putAll (vars.localMap);
- }
-
- /**
- *
- * @param name
- */
- public Data remove( String name )
- {
- return localMap.remove( name );
- }
-
- /**
- *
- */
- public void removeAll()
- {
- localMap.clear();
- }
-
- /**
- *
- * @param d
- * @return
- */
- public boolean hasReferences( Data d )
- {
- return localMap.containsValue(d);
- }
-
- /**
- *
- * @param bo
- * @return
- */
- public boolean hasReferences( LineageObject bo )
- {
- for( Data tmpdat : localMap.values() )
- if ( tmpdat instanceof MatrixObject ) {
- MatrixObject mo = (MatrixObject)tmpdat;
- if( mo.getBroadcastHandle()==bo || mo.getRDDHandle()==bo )
- return true;
- }
- return false;
- }
-
- /**
- *
- * @param d
- * @param earlyAbort
- * @return
- */
- public int getNumReferences( Data d, boolean earlyAbort )
- {
- if ( d == null )
- return 0;
-
- int refCount = 0;
- for( Data tmpdat : localMap.values() )
- if ( tmpdat == d )
- if( ++refCount > 1 && earlyAbort )
- return refCount;
-
- return refCount;
- }
-
- /**
- *
- * @return
- * @throws DMLRuntimeException
- */
- public String serialize()
- throws DMLRuntimeException
- {
- StringBuilder sb = new StringBuilder();
-
- int count = 0;
- for (Entry <String, Data> e : localMap.entrySet ())
- {
- if (count != 0)
- sb.append (ELEMENT_DELIM);
- sb.append (ProgramConverter.serializeDataObject (e.getKey(), e.getValue()));
- count++;
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param varStr
- * @return
- * @throws DMLRuntimeException
- */
- public static LocalVariableMap deserialize(String varStr)
- throws DMLRuntimeException
- {
- StringTokenizer st2 = new StringTokenizer (varStr, ELEMENT_DELIM );
- LocalVariableMap vars = new LocalVariableMap ();
- while( st2.hasMoreTokens() )
- {
- String tmp = st2.nextToken().trim();
- Object[] tmp2 = ProgramConverter.parseDataObject (tmp);
- vars.put ((String) tmp2 [0], (Data) tmp2 [1]);
- }
- return vars;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("Local Variable Map ID = \"");
- sb.append(localID);
- sb.append("\":");
- sb.append(eol);
-
- for (Entry <String, Data> pair : localMap.entrySet()) {
- sb.append(" ");
- sb.append(pair.getKey());
- sb.append(" = ");
- sb.append(pair.getValue());
- sb.append(eol);
- }
-
- return sb.toString();
- }
-
- @Override
- public Object clone()
- {
- return new LocalVariableMap( this );
- }
-}
[06/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
deleted file mode 100644
index a78739c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
+++ /dev/null
@@ -1,707 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-
-/**
- * Remote data partitioner mapper implementation that does the actual
- * partitioning and key creation according to the given format.
- *
- */
-public class DataPartitionerRemoteMapper
- implements Mapper<Writable, Writable, Writable, Writable>
-{
-
- private DataPartitionerMapper _mapper = null;
-
- public DataPartitionerRemoteMapper( )
- {
-
- }
-
- /**
- *
- */
- @Override
- public void map(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _mapper.processKeyValue(key, value, out, reporter);
- }
-
- /**
- *
- */
- @Override
- public void configure(JobConf job)
- {
- long rlen = MRJobConfiguration.getPartitioningNumRows( job );
- long clen = MRJobConfiguration.getPartitioningNumCols( job );
- int brlen = MRJobConfiguration.getPartitioningBlockNumRows( job );
- int bclen = MRJobConfiguration.getPartitioningBlockNumCols( job );
- InputInfo ii = MRJobConfiguration.getPartitioningInputInfo( job );
- OutputInfo oi = MRJobConfiguration.getPartitioningOutputInfo( job );
- PDataPartitionFormat pdf = MRJobConfiguration.getPartitioningFormat( job );
- int n = MRJobConfiguration.getPartitioningSizeN( job );
- boolean keepIndexes = MRJobConfiguration.getPartitioningIndexFlag( job );
-
- if( ii == InputInfo.TextCellInputInfo )
- _mapper = new DataPartitionerMapperTextcell(rlen, clen, brlen, bclen, pdf, n);
- else if( ii == InputInfo.BinaryCellInputInfo )
- _mapper = new DataPartitionerMapperBinarycell(rlen, clen, brlen, bclen, pdf, n);
- else if( ii == InputInfo.BinaryBlockInputInfo )
- {
- if( oi == OutputInfo.BinaryBlockOutputInfo )
- _mapper = new DataPartitionerMapperBinaryblock(rlen, clen, brlen, bclen, pdf, n, keepIndexes);
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- {
- boolean outputEmpty = MRJobConfiguration.getProgramBlocks(job)!=null; //fused parfor
- _mapper = new DataPartitionerMapperBinaryblock2Binarycell(job, rlen, clen, brlen, bclen, pdf, n, keepIndexes, outputEmpty);
- }
- else
- throw new RuntimeException("Partitioning from '"+ii+"' to '"+oi+"' not supported");
- }
- else
- throw new RuntimeException("Unable to configure mapper with unknown input info: "+ii.toString());
- }
-
- /**
- *
- */
- @Override
- public void close()
- throws IOException
- {
- _mapper.close();
- }
-
- private abstract class DataPartitionerMapper //NOTE: could also be refactored as three different mappers
- {
- protected long _rlen = -1;
- protected long _clen = -1;
- protected int _brlen = -1;
- protected int _bclen = -1;
- protected PDataPartitionFormat _pdf = null;
- protected int _n = -1;
-
- protected DataPartitionerMapper( long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pdf, int n )
- {
- _rlen = rlen;
- _clen = clen;
- _brlen = brlen;
- _bclen = bclen;
- _pdf = pdf;
- _n = n;
- }
-
- protected abstract void processKeyValue( Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter )
- throws IOException;
-
- protected void close()
- throws IOException
- {
- //do nothing
- }
- }
-
- private class DataPartitionerMapperTextcell extends DataPartitionerMapper
- {
- private FastStringTokenizer _st = null;
-
- protected DataPartitionerMapperTextcell(long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pdf, int n)
- {
- super(rlen, clen, brlen, bclen, pdf, n);
- _st = new FastStringTokenizer(' ');
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- long row = -1;
- long col = -1;
-
- try
- {
- _st.reset( value.toString() ); //reset tokenizer
- row = _st.nextLong();
- col = _st.nextLong();
- double lvalue = _st.nextDouble();
-
- LongWritable longKey = new LongWritable();
- PairWritableCell pairValue = new PairWritableCell();
- MatrixIndexes key2 = new MatrixIndexes();
- MatrixCell value2 = new MatrixCell();
-
- switch( _pdf )
- {
- case ROW_WISE:
- longKey.set( row );
- row = 1;
- break;
- case ROW_BLOCK_WISE:
- longKey.set( (row-1)/_brlen+1 );
- row = (row-1)%_brlen+1;
- break;
- case ROW_BLOCK_WISE_N:
- longKey.set( (row-1)/_n+1 );
- row = (row-1)%_n+1;
- break;
- case COLUMN_WISE:
- longKey.set( col );
- col = 1;
- break;
- case COLUMN_BLOCK_WISE:
- longKey.set( (col-1)/_bclen+1 );
- col = (col-1)%_bclen+1;
- break;
- case COLUMN_BLOCK_WISE_N:
- longKey.set( (col-1)/_n+1 );
- col = (col-1)%_n+1;
- break;
- default:
- //do nothing
- }
-
- key2.setIndexes(row, col);
- value2.setValue( lvalue );
- pairValue.indexes = key2;
- pairValue.cell = value2;
- out.collect(longKey, pairValue);
- }
- catch (Exception e)
- {
- //post-mortem error handling and bounds checking
- if( row < 1 || row > _rlen || col < 1 || col > _clen )
- {
- throw new IOException("Matrix cell ["+(row)+","+(col)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"].");
- }
- else
- throw new IOException("Unable to partition text cell matrix.", e);
- }
- }
- }
-
- private class DataPartitionerMapperBinarycell extends DataPartitionerMapper
- {
- protected DataPartitionerMapperBinarycell(long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pdf, int n)
- {
- super(rlen, clen, brlen, bclen, pdf, n);
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- long row = -1;
- long col = -1;
-
- try
- {
- LongWritable longKey = new LongWritable();
- PairWritableCell pairValue = new PairWritableCell();
- MatrixIndexes key2 = (MatrixIndexes)key;
- MatrixCell value2 = (MatrixCell)value;
- row = key2.getRowIndex();
- col = key2.getColumnIndex();
-
- switch( _pdf )
- {
- case ROW_WISE:
- longKey.set(row);
- row = 1;
- break;
- case ROW_BLOCK_WISE:
- longKey.set( (row-1)/_brlen+1 );
- row = (row-1)%_brlen+1;
- break;
- case ROW_BLOCK_WISE_N:
- longKey.set( (row-1)/_n+1 );
- row = (row-1)%_n+1;
- break;
- case COLUMN_WISE:
- longKey.set(col);
- col = 1;
- break;
- case COLUMN_BLOCK_WISE:
- longKey.set( (col-1)/_bclen+1 );
- col = (col-1)%_bclen+1;
- break;
- case COLUMN_BLOCK_WISE_N:
- longKey.set( (col-1)/_n+1 );
- col = (col-1)%_n+1;
- break;
- default:
- //do nothing
- }
- key2.setIndexes(row, col);
- pairValue.indexes = key2;
- pairValue.cell = value2;
- out.collect(longKey, pairValue);
- }
- catch (Exception e)
- {
- //post-mortem error handling and bounds checking
- if( row < 1 || row > _rlen || col < 1 || col > _clen )
- {
- throw new IOException("Matrix cell ["+(row)+","+(col)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"].");
- }
- else
- throw new IOException("Unable to partition binary cell matrix.", e);
- }
- }
- }
-
- private class DataPartitionerMapperBinaryblock extends DataPartitionerMapper
- {
- private MatrixBlock _reuseBlk = null;
- private LongWritable _longKey = null;
- private MatrixIndexes _pairKey = null;
- private PairWritableBlock _pair = null;
- private boolean _keepIndexes = false;
-
- protected DataPartitionerMapperBinaryblock(long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pdf, int n, boolean keepIndexes)
- {
- super(rlen, clen, brlen, bclen, pdf, n);
-
- //create reuse keys and block
- _longKey = new LongWritable(); //MR key
- _pair = new PairWritableBlock(); //MR value (pair composed of key and value)
- _pairKey = new MatrixIndexes();
- _reuseBlk = DataPartitioner.createReuseMatrixBlock(pdf, brlen, bclen);
-
- //prewire pair outputs
- _pair.indexes = _pairKey;
- _pair.block = _reuseBlk;
-
- _keepIndexes = keepIndexes;
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- try
- {
- MatrixIndexes key2 = (MatrixIndexes)key;
- MatrixBlock value2 = (MatrixBlock)value;
- long row_offset = (key2.getRowIndex()-1)*_brlen;
- long col_offset = (key2.getColumnIndex()-1)*_bclen;
-
- boolean sparse = value2.isInSparseFormat();
- long nnz = value2.getNonZeros();
- long rows = value2.getNumRows();
- long cols = value2.getNumColumns();
- double sparsity = ((double)nnz)/(rows*cols);
-
- //bound check per block
- if( row_offset + rows < 1 || row_offset + rows > _rlen || col_offset + cols<1 || col_offset + cols > _clen )
- {
- throw new IOException("Matrix block ["+(row_offset+1)+":"+(row_offset+rows)+","+(col_offset+1)+":"+(col_offset+cols)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"].");
- }
-
- //partition inputs according to partitioning scheme
- //(note: output pair is pre-wired, changes only required for blockwise)
- switch( _pdf )
- {
- case ROW_WISE:
- _reuseBlk.reset(1, (int)cols, sparse, (int)(cols*sparsity));
- for( int i=0; i<rows; i++ )
- {
- _longKey.set(row_offset+1+i);
- _pairKey.setIndexes(1, (col_offset/_bclen+1) );
- value2.sliceOperations(i, i, 0, (int)(cols-1), _reuseBlk);
- out.collect(_longKey, _pair);
- _reuseBlk.reset();
- }
- break;
- case ROW_BLOCK_WISE:
- _longKey.set((row_offset/_brlen+1));
- _pairKey.setIndexes(1, (col_offset/_bclen+1) );
- _pair.block = value2;
- out.collect(_longKey, _pair);
- break;
- case ROW_BLOCK_WISE_N:
- _longKey.set((row_offset/_n+1));
- if( _keepIndexes )
- _pairKey.setIndexes(row_offset/_brlen+1, col_offset/_bclen+1 );
- else
- _pairKey.setIndexes(((row_offset%_n)/_brlen)+1, (col_offset/_bclen+1) );
- _pair.block = value2;
- out.collect(_longKey, _pair);
- break;
- case COLUMN_WISE:
- _reuseBlk.reset((int)rows, 1, false);
- for( int i=0; i<cols; i++ )
- {
- _longKey.set(col_offset+1+i);
- _pairKey.setIndexes(row_offset/_brlen+1, 1);
- value2.sliceOperations(0, (int)(rows-1), i, i, _reuseBlk);
- out.collect(_longKey, _pair );
- _reuseBlk.reset();
- }
- break;
- case COLUMN_BLOCK_WISE:
- _longKey.set(col_offset/_bclen+1);
- _pairKey.setIndexes( row_offset/_brlen+1, 1 );
- _pair.block = value2;
- out.collect(_longKey, _pair);
- break;
- case COLUMN_BLOCK_WISE_N:
- _longKey.set(col_offset/_n+1);
- if( _keepIndexes )
- _pairKey.setIndexes( row_offset/_brlen+1, col_offset/_bclen+1 );
- else
- _pairKey.setIndexes( row_offset/_brlen+1, ((col_offset%_n)/_bclen)+1 );
- _pair.block = value2;
- out.collect(_longKey, _pair);
- break;
- default:
- //do nothing
- }
- }
- catch (Exception e)
- {
- throw new IOException("Unable to partition binary block matrix.", e);
- }
- }
- }
-
- /**
- *
- */
- private class DataPartitionerMapperBinaryblock2Binarycell extends DataPartitionerMapper
- {
- private JobConf _cachedJobConf = null;
- private boolean _outputEmpty = false;
-
- private boolean _keepIndexes = false;
-
- private OutputCollector<Writable, Writable> _out = null;
-
- protected DataPartitionerMapperBinaryblock2Binarycell(JobConf job, long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pdf, int n, boolean keepIndexes, boolean outputEmpty)
- {
- super(rlen, clen, brlen, bclen, pdf, n);
- _outputEmpty = outputEmpty;
- _cachedJobConf = job;
-
- _keepIndexes = keepIndexes;
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _out = out;
-
- try
- {
- LongWritable longKey = new LongWritable();
- PairWritableCell pairValue = new PairWritableCell();
- MatrixIndexes key2 = (MatrixIndexes)key;
- MatrixBlock value2 = (MatrixBlock)value;
- MatrixIndexes cellkey2 = new MatrixIndexes();
- MatrixCell cellvalue2 = new MatrixCell();
-
- long row_offset = (key2.getRowIndex()-1)*_brlen;
- long col_offset = (key2.getColumnIndex()-1)*_bclen;
-
- boolean sparse = value2.isInSparseFormat();
- long rows = value2.getNumRows();
- long cols = value2.getNumColumns();
-
- //bound check per block
- if( row_offset + rows < 1 || row_offset + rows > _rlen || col_offset + cols<1 || col_offset + cols > _clen )
- {
- throw new IOException("Matrix block ["+(row_offset+1)+":"+(row_offset+rows)+","+(col_offset+1)+":"+(col_offset+cols)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"].");
- }
-
- long rowBlockIndex = -1;
- long colBlockIndex = -1;
- switch( _pdf )
- {
- case ROW_WISE:
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- longKey.set( row_offset + lcell.i + 1 );
- cellkey2.setIndexes( 1, col_offset + lcell.j + 1 );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int i=0; i<rows; i++ )
- {
- longKey.set(row_offset + i + 1);
- for( int j=0; j<cols; j++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( 1, col_offset + j + 1 );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- }
- break;
- case ROW_BLOCK_WISE:
- longKey.set((row_offset/_brlen+1));
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- cellkey2.setIndexes( 1, col_offset + lcell.j + 1 );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( 1, col_offset + j + 1 );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- break;
- case ROW_BLOCK_WISE_N:
- longKey.set((row_offset/_n+1));
- if( _keepIndexes )
- rowBlockIndex = ((row_offset)/_brlen)+1;
- else
- rowBlockIndex = ((row_offset%_n)/_brlen)+1;
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- cellkey2.setIndexes( rowBlockIndex, col_offset + lcell.j + 1 );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( rowBlockIndex, col_offset + j + 1 );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- break;
- case COLUMN_WISE:
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- longKey.set( col_offset + lcell.j + 1 );
- cellkey2.setIndexes( row_offset + lcell.i + 1, 1 );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int j=0; j<cols; j++ )
- {
- longKey.set(col_offset + j + 1);
- for( int i=0; i<rows; i++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( row_offset + i + 1, 1 );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- }
- break;
- case COLUMN_BLOCK_WISE:
- longKey.set(col_offset/_bclen+1);
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- cellkey2.setIndexes( row_offset + lcell.i + 1, 1 );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int j=0; j<cols; j++ )
- for( int i=0; i<rows; i++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( row_offset + i + 1, 1 );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- break;
- case COLUMN_BLOCK_WISE_N:
- longKey.set(col_offset/_n+1);
- if( _keepIndexes )
- colBlockIndex = ((col_offset)/_bclen)+1;
- else
- colBlockIndex = ((col_offset%_n)/_bclen)+1;
- if( sparse )
- {
- SparseRowsIterator iter = value2.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- cellkey2.setIndexes( row_offset + lcell.i + 1, colBlockIndex );
- cellvalue2.setValue( lcell.v );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- else
- for( int j=0; j<cols; j++ )
- for( int i=0; i<rows; i++ )
- {
- double lvalue = value2.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //only for nnz
- {
- cellkey2.setIndexes( row_offset + i + 1, colBlockIndex );
- cellvalue2.setValue( lvalue );
- pairValue.indexes = cellkey2;
- pairValue.cell = cellvalue2;
- out.collect(longKey, pairValue);
- }
- }
- break;
- default:
- //do nothing
- }
- }
- catch (Exception e)
- {
- throw new IOException("Unable to partition binary block matrix.", e);
- }
- }
-
- @Override
- protected void close()
- throws IOException
- {
- if( _outputEmpty )
- {
- LongWritable longKey = new LongWritable();
- PairWritableCell pairValue = new PairWritableCell();
- pairValue.indexes = new MatrixIndexes(-1,-1);
- pairValue.cell = new MatrixCell(0);
-
- long mapID = Long.parseLong(MapReduceTool.getUniqueKeyPerTask(_cachedJobConf, true));
- long numMap = _cachedJobConf.getNumMapTasks();
-
- //output part of empty blocks (all mappers contribute for better load balance),
- //where mapper responsibility is distributed over all partitions
- long numPartitions = -1;
- switch( _pdf ){
- case ROW_WISE: numPartitions = _rlen; break;
- case ROW_BLOCK_WISE: numPartitions = (int)Math.ceil(_rlen/(double)_brlen); break;
- case COLUMN_WISE: numPartitions = _clen; break;
- case COLUMN_BLOCK_WISE: numPartitions = (int)Math.ceil(_clen/(double)_bclen); break;
- default:
- //do nothing
- }
-
- long len = (long)Math.ceil((double)numPartitions/numMap);
- long start = mapID * len;
- long end = Math.min((mapID+1) * len, numPartitions);
- for( long i=start; i<end; i++ )
- {
- longKey.set( i+1 );
- _out.collect(longKey, pairValue);
- }
- }
-
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
deleted file mode 100644
index 27d858f..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-
-/**
- * Remote data partitioner reducer implementation that takes the given
- * partitions and writes it to individual files. The reasons for the design
- * with a reducer instead of MultipleOutputs are
- * 1) robustness wrt num open file descriptors, outofmem
- * 2) no append for sequence files
- * 3) performance of actual write
- *
- */
-public class DataPartitionerRemoteReducer
- implements Reducer<LongWritable, Writable, Writable, Writable>
-{
-
- private DataPartitionerReducer _reducer = null;
-
- public DataPartitionerRemoteReducer( )
- {
-
- }
-
- @Override
- public void reduce(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _reducer.processKeyValueList(key, valueList, out, reporter);
- }
-
- /**
- *
- */
- public void configure(JobConf job)
- {
- String fnameNew = MRJobConfiguration.getPartitioningFilename( job );
- OutputInfo oi = MRJobConfiguration.getPartitioningOutputInfo( job );
-
- if( oi == OutputInfo.TextCellOutputInfo )
- _reducer = new DataPartitionerReducerTextcell(job, fnameNew);
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- _reducer = new DataPartitionerReducerBinarycell(job, fnameNew);
- else if( oi == OutputInfo.BinaryBlockOutputInfo )
- _reducer = new DataPartitionerReducerBinaryblock(job, fnameNew);
- else
- throw new RuntimeException("Unable to configure reducer with unknown output info: "+oi.toString());
- }
-
- /**
- *
- */
- @Override
- public void close() throws IOException
- {
- //do nothing
- }
-
-
- private abstract class DataPartitionerReducer //NOTE: could also be refactored as three different reducers
- {
- protected JobConf _job = null;
- protected FileSystem _fs = null;
- protected String _fnameNew = null;
-
- protected DataPartitionerReducer( JobConf job, String fnameNew )
- {
- _job = job;
- _fnameNew = fnameNew;
-
- try {
- _fs = FileSystem.get(_job);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected abstract void processKeyValueList( LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter )
- throws IOException;
- }
-
- private class DataPartitionerReducerTextcell extends DataPartitionerReducer
- {
- private StringBuilder _sb = null;
-
- protected DataPartitionerReducerTextcell( JobConf job, String fnameNew )
- {
- super(job, fnameNew);
-
- //for obj reuse and preventing repeated buffer re-allocations
- _sb = new StringBuilder();
- }
-
- @Override
- protected void processKeyValueList(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
-
- BufferedWriter writer = null;
- try
- {
- Path path = new Path(_fnameNew+"/"+key.get());
- writer = new BufferedWriter(new OutputStreamWriter(_fs.create(path,true)));
-
- while( valueList.hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.next();
-
- _sb.append(pairValue.indexes.getRowIndex());
- _sb.append(' ');
- _sb.append(pairValue.indexes.getColumnIndex());
- _sb.append(' ');
- _sb.append(pairValue.cell.getValue());
- _sb.append('\n');
- writer.write(_sb.toString());
-
- _sb.setLength(0);
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- }
-
-
- private class DataPartitionerReducerBinarycell extends DataPartitionerReducer
- {
- protected DataPartitionerReducerBinarycell( JobConf job, String fnameNew )
- {
- super(job, fnameNew);
- }
-
- @Override
- @SuppressWarnings("deprecation")
- protected void processKeyValueList(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- SequenceFile.Writer writer = null;
- try
- {
- Path path = new Path(_fnameNew+"/"+key.get());
- writer = new SequenceFile.Writer(_fs, _job, path, MatrixIndexes.class, MatrixCell.class);
- while( valueList.hasNext() )
- {
- PairWritableCell pairValue = (PairWritableCell)valueList.next();
- writer.append(pairValue.indexes, pairValue.cell);
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- }
-
- private class DataPartitionerReducerBinaryblock extends DataPartitionerReducer
- {
- protected DataPartitionerReducerBinaryblock( JobConf job, String fnameNew )
- {
- super(job, fnameNew);
- }
-
- @Override
- @SuppressWarnings("deprecation")
- protected void processKeyValueList(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- SequenceFile.Writer writer = null;
- try
- {
- Path path = new Path(_fnameNew+"/"+key.get());
- writer = new SequenceFile.Writer(_fs, _job, path, MatrixIndexes.class, MatrixBlock.class);
- while( valueList.hasNext() )
- {
- PairWritableBlock pairValue = (PairWritableBlock)valueList.next();
- writer.append(pairValue.indexes, pairValue.block);
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
deleted file mode 100644
index 41bda63..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import org.apache.spark.api.java.JavaPairRDD;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * MR job class for submitting parfor remote partitioning MR jobs.
- *
- */
-public class DataPartitionerRemoteSpark extends DataPartitioner
-{
-
- //private boolean _keepIndexes = false;
- private ExecutionContext _ec = null;
- private long _numRed = -1;
-
- public DataPartitionerRemoteSpark(PDataPartitionFormat dpf, int n, ExecutionContext ec, long numRed, boolean keepIndexes)
- {
- super(dpf, n);
-
- _ec = ec;
- _numRed = numRed;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- protected void partitionMatrix(MatrixObject in, String fnameNew, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen)
- throws DMLRuntimeException
- {
- String jobname = "ParFor-DPSP";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- SparkExecutionContext sec = (SparkExecutionContext)_ec;
-
- try
- {
- //cleanup existing output files
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- //determine degree of parallelism
- int numRed = (int)determineNumReducers(rlen, clen, brlen, bclen, _numRed);
-
- //get input rdd
- JavaPairRDD<MatrixIndexes, MatrixBlock> inRdd = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- sec.getRDDHandleForMatrixObject(in, InputInfo.BinaryBlockInputInfo);
- MatrixCharacteristics mc = in.getMatrixCharacteristics();
-
- //run spark remote data partition job
- DataPartitionerRemoteSparkMapper dpfun = new DataPartitionerRemoteSparkMapper(mc, ii, oi, _format);
- DataPartitionerRemoteSparkReducer wfun = new DataPartitionerRemoteSparkReducer(fnameNew, oi);
- inRdd.flatMapToPair(dpfun) //partition the input blocks
- .groupByKey(numRed) //group partition blocks
- .foreach( wfun ); //write partitions to hdfs
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //maintain statistics
- Statistics.incrementNoOfCompiledSPInst();
- Statistics.incrementNoOfExecutedSPInst();
- if( DMLScript.STATISTICS ){
- Statistics.maintainCPHeavyHitters(jobname, System.nanoTime()-t0);
- }
- }
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param numRed
- * @return
- */
- private long determineNumReducers(long rlen, long clen, int brlen, int bclen, long numRed)
- {
- //set the number of mappers and reducers
- long reducerGroups = -1;
- switch( _format )
- {
- case ROW_WISE: reducerGroups = rlen; break;
- case COLUMN_WISE: reducerGroups = clen; break;
- case ROW_BLOCK_WISE: reducerGroups = (rlen/brlen)+((rlen%brlen==0)?0:1); break;
- case COLUMN_BLOCK_WISE: reducerGroups = (clen/bclen)+((clen%bclen==0)?0:1); break;
- case ROW_BLOCK_WISE_N: reducerGroups = (rlen/_n)+((rlen%_n==0)?0:1); break;
- case COLUMN_BLOCK_WISE_N: reducerGroups = (clen/_n)+((clen%_n==0)?0:1); break;
- default:
- //do nothing
- }
-
- return (int)Math.min( numRed, reducerGroups);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
deleted file mode 100644
index 6210c94..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-import scala.Tuple2;
-
-/**
- * NOTE: for the moment we only support binary block here
- * TODO extend impl for binarycell and textcell
- *
- * Interface of Writable output in order to support both PairWritableBlock and PairWritableCell.
- *
- */
-public class DataPartitionerRemoteSparkMapper extends ParWorker implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, Long, Writable>
-{
-
- private static final long serialVersionUID = 332887624852010957L;
-
- private long _rlen = -1;
- private long _clen = -1;
- private long _brlen = -1;
- private long _bclen = -1;
- private long _n = -1;
- //private InputInfo _ii = null;
- //private OutputInfo _oi = null;
- private PDataPartitionFormat _dpf = null;
-
- public DataPartitionerRemoteSparkMapper(MatrixCharacteristics mc, InputInfo ii, OutputInfo oi, PDataPartitionFormat dpf)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- _rlen = mc.getRows();
- _clen = mc.getCols();
- _brlen = mc.getRowsPerBlock();
- _bclen = mc.getColsPerBlock();
- //_ii = ii;
- //_oi = oi;
- _dpf = dpf;
- }
-
-
- @Override
- public Iterable<Tuple2<Long, Writable>> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
- throws Exception
- {
- List<Tuple2<Long, Writable>> ret = new LinkedList<Tuple2<Long, Writable>>();
-
- MatrixIndexes key2 = arg0._1();
- MatrixBlock value2 = arg0._2();
- long row_offset = (key2.getRowIndex()-1)*_brlen;
- long col_offset = (key2.getColumnIndex()-1)*_bclen;
- long rows = value2.getNumRows();
- long cols = value2.getNumColumns();
-
- //bound check per block
- if( row_offset + rows < 1 || row_offset + rows > _rlen || col_offset + cols<1 || col_offset + cols > _clen )
- {
- throw new IOException("Matrix block ["+(row_offset+1)+":"+(row_offset+rows)+","+(col_offset+1)+":"+(col_offset+cols)+"] " +
- "out of overall matrix range [1:"+_rlen+",1:"+_clen+"].");
- }
-
- //partition inputs according to partitioning scheme
- switch( _dpf )
- {
- case ROW_WISE: {
- MatrixBlock[] blks = DataConverter.convertToMatrixBlockPartitions(value2, false);
- for( int i=0; i<rows; i++ ) {
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(1, col_offset/_bclen+1);
- tmp.block = blks[i];
- ret.add(new Tuple2<Long,Writable>(new Long(row_offset+1+i),tmp));
- }
- break;
- }
- case ROW_BLOCK_WISE: {
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(1, col_offset/_bclen+1);
- tmp.block = new MatrixBlock(value2);
- ret.add(new Tuple2<Long,Writable>(new Long(row_offset/_brlen+1),tmp));
- break;
- }
- case ROW_BLOCK_WISE_N:{
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(((row_offset%_n)/_brlen)+1, col_offset/_bclen+1);
- tmp.block = new MatrixBlock(value2);
- ret.add(new Tuple2<Long,Writable>(new Long(row_offset/_n+1),tmp));
- break;
- }
- case COLUMN_WISE:{
- MatrixBlock[] blks = DataConverter.convertToMatrixBlockPartitions(value2, true);
- for( int i=0; i<cols; i++ ) {
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(row_offset/_brlen+1, 1);
- tmp.block = blks[i];
- ret.add(new Tuple2<Long,Writable>(new Long(col_offset+1+i),tmp));
- }
- break;
- }
- case COLUMN_BLOCK_WISE: {
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(row_offset/_brlen+1, 1);
- tmp.block = new MatrixBlock(value2);
- ret.add(new Tuple2<Long,Writable>(new Long(col_offset/_bclen+1),tmp));
- break;
- }
- case COLUMN_BLOCK_WISE_N: {
- PairWritableBlock tmp = new PairWritableBlock();
- tmp.indexes = new MatrixIndexes(row_offset/_brlen+1, ((col_offset%_n)/_bclen)+1);
- tmp.block = new MatrixBlock(value2);
- ret.add(new Tuple2<Long,Writable>(new Long(col_offset/_n+1),tmp));
- break;
- }
-
- default:
- throw new DMLRuntimeException("Unsupported partition format: "+_dpf);
- }
-
- return ret;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
deleted file mode 100644
index 8207d8a..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.File;
-import java.util.Iterator;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.spark.api.java.function.VoidFunction;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-
-import scala.Tuple2;
-
-/**
- *
- */
-public class DataPartitionerRemoteSparkReducer implements VoidFunction<Tuple2<Long, Iterable<Writable>>>
-{
-
- private static final long serialVersionUID = -7149865018683261964L;
-
- private String _fnameNew = null;
-
- public DataPartitionerRemoteSparkReducer(String fnameNew, OutputInfo oi)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- _fnameNew = fnameNew;
- //_oi = oi;
- }
-
- @Override
- @SuppressWarnings("deprecation")
- public void call(Tuple2<Long, Iterable<Writable>> arg0)
- throws Exception
- {
- //prepare grouped partition input
- Long key = arg0._1();
- Iterator<Writable> valueList = arg0._2().iterator();
-
- //write entire partition to binary block sequence file
- SequenceFile.Writer writer = null;
- try
- {
- Configuration job = new Configuration(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(_fnameNew + File.separator + key);
- writer = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixBlock.class);
- while( valueList.hasNext() )
- {
- PairWritableBlock pair = (PairWritableBlock) valueList.next();
- writer.append(pair.indexes, pair.block);
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalParWorker.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalParWorker.java
deleted file mode 100644
index 08c4733..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalParWorker.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.Collection;
-
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-
-/**
- * Instances of this class can be used to execute tasks in parallel. Within each ParWorker
- * multiple iterations of a single task and subsequent tasks are executed sequentially.
- *
- * Resiliency approach: retry on computation error, abort on task queue error
- *
- *
- */
-public class LocalParWorker extends ParWorker implements Runnable
-{
- protected LocalTaskQueue<Task> _taskQueue = null;
-
- protected Collection<String> _fnNames = null;
-
- protected boolean _stopped = false;
- protected int _max_retry = -1;
-
- public LocalParWorker( long ID, LocalTaskQueue<Task> q, ParForBody body, int max_retry, boolean monitor )
- {
- super(ID, body, monitor);
-
- _taskQueue = q;
- _stopped = false;
-
- _max_retry = max_retry;
- }
-
- /**
- * Sets the status to stopped such that execution will be aborted as soon as the
- * current task is finished.
- */
- public void setStopped() {
- _stopped = true;
- }
-
- public void setFunctionNames(Collection<String> fnNames) {
- _fnNames = fnNames;
- }
-
- public Collection<String> getFunctionNames() {
- return _fnNames;
- }
-
- @Override
- public void run()
- {
- // monitoring start
- Timing time1 = ( _monitor ? new Timing(true) : null );
-
- //setup fair scheduler pool for worker thread
- if( OptimizerUtils.isSparkExecutionMode() ) {
- SparkExecutionContext sec = (SparkExecutionContext)_ec;
- sec.setThreadLocalSchedulerPool("parforPool"+_workerID);
- }
-
- // continuous execution (execute tasks until (1) stopped or (2) no more tasks)
- Task lTask = null;
-
- while( !_stopped )
- {
- //dequeue the next task (abort on NO_MORE_TASKS or error)
- try
- {
- lTask = _taskQueue.dequeueTask();
-
- if( lTask == LocalTaskQueue.NO_MORE_TASKS ) // task queue closed (no more tasks)
- break; //normal end of parallel worker
- }
- catch(Exception ex)
- {
- // abort on taskqueue error
- LOG.warn("Error reading from task queue: "+ex.getMessage());
- LOG.warn("Stopping LocalParWorker.");
- break; //no exception thrown to prevent blocking on join
- }
-
- //execute the task sequentially (re-try on error)
- boolean success = false;
- int retrys = _max_retry;
-
- while( !success )
- {
- try
- {
- ///////
- //core execution (see ParWorker)
- executeTask( lTask );
- success = true;
- }
- catch (Exception ex)
- {
- LOG.error("Failed to execute "+lTask.toString()+", retry:"+retrys, ex);
-
- if( retrys > 0 )
- retrys--; //retry on task error
- else
- {
- // abort on no remaining retrys
- LOG.error("Error executing task: ",ex);
- LOG.error("Stopping LocalParWorker.");
- break; //no exception thrown to prevent blocking on join
- }
- }
- }
- }
-
- //setup fair scheduler pool for worker thread
- if( OptimizerUtils.isSparkExecutionMode() ) {
- SparkExecutionContext sec = (SparkExecutionContext)_ec;
- sec.cleanupThreadLocalSchedulerPool();
- }
-
- if( _monitor ) {
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_NUMTASKS, _numTasks);
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_NUMITERS, _numIters);
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_EXEC_T, time1.stop());
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalTaskQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalTaskQueue.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalTaskQueue.java
deleted file mode 100644
index 859044a..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/LocalTaskQueue.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.LinkedList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * This class provides a way of dynamic task distribution to multiple workers
- * in local multi-threaded environments. Its main purpose is inter-thread communication
- * for achieving dynamic load balancing. A good load balance between parallel workers is crucial
- * with regard to the overall speedup of parallelization (see Amdahl's law).
- *
- * From a technical perspective, a thread monitor concept is used for blocking of waiting writers
- * and readers. Synchronized writes and reads ensure that each task is only read by exactly one
- * reader. Furthermore, the queue is implemented as a simple FIFO.
- *
- *
- */
-public class LocalTaskQueue<T>
-{
-
- public static final int MAX_SIZE = 100000; //main memory constraint
- public static final Object NO_MORE_TASKS = null; //object to signal NO_MORE_TASKS
-
- private LinkedList<T> _data = null;
- private boolean _closedInput = false;
- private static final Log LOG = LogFactory.getLog(LocalTaskQueue.class.getName());
-
- public LocalTaskQueue()
- {
- _data = new LinkedList<T>();
- _closedInput = false;
- }
-
- /**
- * Synchronized insert of a new task to the end of the FIFO queue.
- *
- * @param t
- * @throws InterruptedException
- */
- public synchronized void enqueueTask( T t )
- throws InterruptedException
- {
- while( _data.size() + 1 > MAX_SIZE )
- {
- LOG.warn("MAX_SIZE of task queue reached.");
- wait(); //max constraint reached, wait for read
- }
-
- _data.addLast( t );
-
- notify(); //notify waiting readers
- }
-
- /**
- * Synchronized read and delete from the top of the FIFO queue.
- *
- * @return
- * @throws InterruptedException
- */
- @SuppressWarnings("unchecked")
- public synchronized T dequeueTask()
- throws InterruptedException
- {
- while( _data.isEmpty() )
- {
- if( !_closedInput )
- wait(); // wait for writers
- else
- return (T)NO_MORE_TASKS;
- }
-
- T t = _data.removeFirst();
-
- notify(); // notify waiting writers
-
- return t;
- }
-
- /**
- * Synchronized (logical) insert of a NO_MORE_TASKS symbol at the end of the FIFO queue in order to
- * mark that no more tasks will be inserted into the queue.
- */
- public synchronized void closeInput()
- {
- _closedInput = true;
- notifyAll(); //notify all waiting readers
- }
-
- /**
- * Synchronized read of the current number of tasks in the queue.
- *
- * @return
- * @throws InterruptedException
- */
- public synchronized int size()
- throws InterruptedException
- {
- return _data.size();
- }
-
- @Override
- public synchronized String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("TASK QUEUE (size=");
- sb.append(_data.size());
- sb.append(",close=");
- sb.append(_closedInput);
- sb.append(")\n");
-
- int count = 1;
- for( T t : _data )
- {
- sb.append(" TASK #");
- sb.append(count);
- sb.append(": ");
- sb.append(t.toString());
- sb.append("\n");
-
- count++;
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParForBody.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParForBody.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParForBody.java
deleted file mode 100644
index d60e72b..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParForBody.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-
-/**
- * Wrapper for exchanging parfor body data structures.
- *
- */
-public class ParForBody
-{
-
-
- private ArrayList<String> _resultVarNames;
- private ArrayList<ProgramBlock> _childBlocks;
- private ExecutionContext _ec;
-
- public ParForBody()
- {
-
- }
-
- public ParForBody( ArrayList<ProgramBlock> childBlocks,
- ArrayList<String> resultVarNames, ExecutionContext ec )
- {
- _resultVarNames = resultVarNames;
- _childBlocks = childBlocks;
- _ec = ec;
- }
-
- public LocalVariableMap getVariables()
- {
- return _ec.getVariables();
- }
-
- public ArrayList<String> getResultVarNames()
- {
- return _resultVarNames;
- }
-
- public void setResultVarNames(ArrayList<String> resultVarNames)
- {
- _resultVarNames = resultVarNames;
- }
-
- public ArrayList<ProgramBlock> getChildBlocks()
- {
- return _childBlocks;
- }
-
- public void setChildBlocks(ArrayList<ProgramBlock> childBlocks)
- {
- _childBlocks = childBlocks;
- }
-
- public ExecutionContext getEc()
- {
- return _ec;
- }
-
- public void setEc(ExecutionContext ec)
- {
- _ec = ec;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParWorker.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParWorker.java
deleted file mode 100644
index 6be0527..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ParWorker.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * Super class for master/worker pattern implementations. Central place to
- * execute set or range tasks.
- *
- */
-public abstract class ParWorker
-{
-
- protected static final Log LOG = LogFactory.getLog(ParWorker.class.getName());
-
- protected long _workerID = -1;
-
- protected ArrayList<ProgramBlock> _childBlocks = null;
- protected ExecutionContext _ec = null;
- protected ArrayList<String> _resultVars = null;
-
- protected boolean _monitor = false;
-
- protected long _numTasks = -1;
- protected long _numIters = -1;
-
- public ParWorker()
- {
- //implicit constructor (required if parameters not known on object creation,
- //e.g., RemoteParWorkerMapper)
- }
-
- public ParWorker( long ID, ParForBody body, boolean monitor )
- {
- _workerID = ID;
-
- if( body != null )
- {
- _childBlocks = body.getChildBlocks();
- _ec = body.getEc();
- _resultVars = body.getResultVarNames();
- }
-
- _monitor = monitor;
-
- _numTasks = 0;
- _numIters = 0;
- }
-
- /**
- *
- * @return
- */
- public LocalVariableMap getVariables()
- {
- return _ec.getVariables();
- }
-
- /**
- * Returns a summary statistic of executed tasks and hence should only be called
- * after execution.
- *
- * @return
- */
- public long getExecutedTasks()
- {
- return _numTasks;
- }
-
- /**
- * Returns a summary statistic of executed iterations and hence should only be called
- * after execution.
- *
- * @return
- */
- public long getExecutedIterations()
- {
- return _numIters;
- }
-
- /**
- *
- */
- public void resetExecutedTasks()
- {
- _numTasks = 0;
- _numIters = 0;
- }
-
- /**
- *
- */
- protected void pinResultVariables()
- {
- for( String var : _resultVars )
- {
- Data dat = _ec.getVariable(var);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject)dat;
- mo.enableCleanup(false);
- }
- }
- }
-
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- protected void executeTask( Task task )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- LOG.trace("EXECUTE PARFOR_WORKER ID="+_workerID+" for task "+task.toCompactString());
-
- switch( task.getType() )
- {
- case SET:
- executeSetTask( task );
- break;
- case RANGE:
- executeRangeTask( task );
- break;
- }
- }
-
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void executeSetTask( Task task )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //monitoring start
- Timing time1 = null, time2 = null;
- if( _monitor )
- {
- time1 = new Timing(true);
- time2 = new Timing(true);
- }
-
- //core execution
-
- //foreach iteration in task, execute iteration body
- for( IntObject indexVal : task.getIterations() )
- {
- //System.out.println(" EXECUTE ITERATION: "+indexVal.getName()+"="+indexVal.getIntValue());
-
- //set index values
- _ec.setVariable(indexVal.getName(), indexVal);
-
- // for each program block
- for (ProgramBlock pb : _childBlocks)
- pb.execute(_ec);
-
- _numIters++;
-
- if( _monitor )
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_ITER_T, time1.stop());
- }
-
- _numTasks++;
-
- //monitoring end
- if( _monitor )
- {
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASKSIZE, task.size());
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASK_T, time2.stop());
- }
- }
-
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void executeRangeTask( Task task )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //monitoring start
- Timing time1 = null, time2 = null;
- if( _monitor )
- {
- time1 = new Timing(true);
- time2 = new Timing(true);
- }
-
- //core execution
- List<IntObject> tmp = task.getIterations();
- String lVarName = tmp.get(0).getName();
- long lFrom = tmp.get(0).getLongValue();
- long lTo = tmp.get(1).getLongValue();
- long lIncr = tmp.get(2).getLongValue();
-
- for( long i=lFrom; i<=lTo; i+=lIncr )
- {
- //set index values
- _ec.setVariable(lVarName, new IntObject(lVarName,i));
-
- // for each program block
- for (ProgramBlock pb : _childBlocks)
- pb.execute(_ec);
-
- _numIters++;
-
- if( _monitor )
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_ITER_T, time1.stop());
- }
-
- _numTasks++;
-
- //monitoring end
- if( _monitor )
- {
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASKSIZE, task.size());
- StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASK_T, time2.stop());
- }
- }
-
-}
-
-
[18/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/IfStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/IfStatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/IfStatementBlock.java
deleted file mode 100644
index 4de36e5..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/IfStatementBlock.java
+++ /dev/null
@@ -1,537 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-
-
-public class IfStatementBlock extends StatementBlock
-{
-
- private Hop _predicateHops;
- private Lop _predicateLops = null;
- private boolean _requiresPredicateRecompile = false;
-
- @Override
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
- if (_statements.size() > 1){
- raiseValidateError("IfStatementBlock should only have 1 statement (IfStatement)", conditional);
- }
-
- IfStatement ifstmt = (IfStatement) _statements.get(0);
-
- //validate conditional predicate (incl constant propagation)
- Expression pred = ifstmt.getConditionalPredicate().getPredicate();
- pred.validateExpression(ids.getVariables(), constVars, conditional);
- if( pred instanceof DataIdentifier && constVars.containsKey( ((DataIdentifier)pred).getName()) ) {
- ifstmt.getConditionalPredicate().setPredicate(constVars.get(((DataIdentifier)pred).getName()));
- }
-
- HashMap<String,ConstIdentifier> constVarsIfCopy = new HashMap<String,ConstIdentifier>(constVars);
- HashMap<String,ConstIdentifier> constVarsElseCopy = new HashMap<String,ConstIdentifier> (constVars);
-
- VariableSet idsIfCopy = new VariableSet(ids);
- VariableSet idsElseCopy = new VariableSet(ids);
- VariableSet idsOrigCopy = new VariableSet(ids);
-
- // handle if stmt body
- _dmlProg = dmlProg;
- ArrayList<StatementBlock> ifBody = ifstmt.getIfBody();
- for(StatementBlock sb : ifBody){ //conditional exec
- idsIfCopy = sb.validate(dmlProg, idsIfCopy, constVarsIfCopy, true);
- constVarsIfCopy = sb.getConstOut();
- }
-
- // handle else stmt body
- ArrayList<StatementBlock> elseBody = ifstmt.getElseBody();
- for(StatementBlock sb : elseBody){ //conditional exec
- idsElseCopy = sb.validate(dmlProg,idsElseCopy, constVarsElseCopy, true);
- constVarsElseCopy = sb.getConstOut();
- }
-
-
- /////////////////////////////////////////////////////////////////////////////////
- // check data type and value type are same for updated variables in both
- // if statement and else statement
- // (reject conditional data type change)
- /////////////////////////////////////////////////////////////////////////////////
- for (String updatedVar : this._updated.getVariableNames()){
- DataIdentifier origVersion = idsOrigCopy.getVariable(updatedVar);
- DataIdentifier ifVersion = idsIfCopy.getVariable(updatedVar);
- DataIdentifier elseVersion = idsElseCopy.getVariable(updatedVar);
-
- //data type handling: reject conditional data type change
- if( ifVersion != null && elseVersion != null ) //both branches exist
- {
- if (!ifVersion.getOutput().getDataType().equals(elseVersion.getOutput().getDataType())){
- raiseValidateError("IfStatementBlock has unsupported conditional data type change of variable '"+updatedVar+"' in if/else branch.", conditional);
- }
- }
- else if( origVersion !=null ) //only if branch exists
- {
- if (!ifVersion.getOutput().getDataType().equals(origVersion.getOutput().getDataType())){
- raiseValidateError("IfStatementBlock has unsupported conditional data type change of variable '"+updatedVar+"' in if branch.", conditional);
- }
- }
-
- //value type handling
- if (ifVersion != null && elseVersion != null && !ifVersion.getOutput().getValueType().equals(elseVersion.getOutput().getValueType())){
- LOG.warn(elseVersion.printWarningLocation() + "Variable " + elseVersion.getName() + " defined with different value type in if and else clause.");
- }
- }
-
- // handle constant variable propagation -- (IF UNION ELSE) MINUS updated vars
-
- //////////////////////////////////////////////////////////////////////////////////
- // handle constant variables
- // 1) (IF UNION ELSE) MINUS updated const vars
- // 2) reconcile updated const vars
- // a) IF updated const variables have same value and datatype in both if / else branch, THEN set updated size to updated size
- // b) ELSE leave out of reconciled set
- /////////////////////////////////////////////////////////////////////////////////
-
- HashMap<String,ConstIdentifier> recConstVars = new HashMap<String,ConstIdentifier>();
-
- // STEP 1: (IF UNION ELSE) MINUS updated vars
- for (Entry<String,ConstIdentifier> e : constVarsIfCopy.entrySet() ){
- String varName = e.getKey();
- if (!_updated.containsVariable(varName))
- recConstVars.put(varName, e.getValue());
- }
- for (Entry<String,ConstIdentifier> e : constVarsElseCopy.entrySet() ){
- String varName = e.getKey();
- if (!_updated.containsVariable(varName))
- recConstVars.put(varName, e.getValue());
- }
-
-
- // STEP 2: check that updated const values have in both if / else branches
- // a) same data type,
- // b) same value type (SCALAR),
- // c) same value
- for (String updatedVar : _updated.getVariableNames()){
- DataIdentifier ifVersion = idsIfCopy.getVariable(updatedVar);
- DataIdentifier elseVersion = idsElseCopy.getVariable(updatedVar);
-
- if (ifVersion != null && elseVersion != null
- && ifVersion.getOutput().getDataType().equals(DataType.SCALAR)
- && elseVersion.getOutput().getDataType().equals(DataType.SCALAR)
- && ifVersion.getOutput().getValueType().equals(elseVersion.getOutput().getValueType()))
- {
- ConstIdentifier ifConstVersion = constVarsIfCopy.get(updatedVar);
- ConstIdentifier elseConstVersion = constVarsElseCopy.get(updatedVar);
- // IntIdentifier
- if (ifConstVersion != null && elseConstVersion != null && ifConstVersion instanceof IntIdentifier && elseConstVersion instanceof IntIdentifier){
- if ( ((IntIdentifier)ifConstVersion).getValue() == ((IntIdentifier) elseConstVersion).getValue() )
- recConstVars.put(updatedVar, ifConstVersion);
- }
- // DoubleIdentifier
- else if (ifConstVersion != null && elseConstVersion != null && ifConstVersion instanceof DoubleIdentifier && elseConstVersion instanceof DoubleIdentifier){
- if ( ((DoubleIdentifier)ifConstVersion).getValue() == ((DoubleIdentifier) elseConstVersion).getValue() )
- recConstVars.put(updatedVar, ifConstVersion);
- }
- // Boolean
- else if (ifConstVersion != null && elseConstVersion != null && ifConstVersion instanceof BooleanIdentifier && elseConstVersion instanceof BooleanIdentifier){
- if ( ((BooleanIdentifier)ifConstVersion).getValue() == ((BooleanIdentifier) elseConstVersion).getValue() )
- recConstVars.put(updatedVar, ifConstVersion);
- }
-
- // String
- else if (ifConstVersion != null && elseConstVersion != null && ifConstVersion instanceof StringIdentifier && elseConstVersion instanceof StringIdentifier){
- if ( ((StringIdentifier)ifConstVersion).getValue().equals(((StringIdentifier) elseConstVersion).getValue()) )
- recConstVars.put(updatedVar, ifConstVersion);
- }
- }
- }
-
- //////////////////////////////////////////////////////////////////////////////////
- // handle DataIdentifier variables
- // 1) (IF UNION ELSE) MINUS updated vars
- // 2) reconcile size updated variables
- // a) IF updated variables have same size in both if / else branch, THEN set updated size to updated size
- // b) ELSE set size updated to (-1,-1)
- // 3) add updated vars to reconciled set
- /////////////////////////////////////////////////////////////////////////////////
-
- // STEP 1: (IF UNION ELSE) MINUS updated vars
- VariableSet recVars = new VariableSet();
-
- for (String varName : idsIfCopy.getVariableNames()){
- if (!_updated.containsVariable(varName))
- recVars.addVariable(varName,idsIfCopy.getVariable(varName));
- }
- for (String varName : idsElseCopy.getVariableNames()){
- if (!_updated.containsVariable(varName))
- recVars.addVariable(varName,idsElseCopy.getVariable(varName));
- }
-
- // STEP 2: reconcile size of updated variables
- for (String updatedVar : _updated.getVariableNames()){
- DataIdentifier ifVersion = idsIfCopy.getVariable(updatedVar);
- DataIdentifier elseVersion = idsElseCopy.getVariable(updatedVar);
- DataIdentifier origVersion = idsOrigCopy.getVariable(updatedVar);
-
- if (ifVersion != null && elseVersion != null) {
- long updatedDim1 = -1, updatedDim2 = -1;
- long updatedNnz = -1;
-
- long ifVersionDim1 = (ifVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)ifVersion).getOrigDim1() : ifVersion.getDim1();
- long elseVersionDim1 = (elseVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)elseVersion).getOrigDim1() : elseVersion.getDim1();
-
- long ifVersionDim2 = (ifVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)ifVersion).getOrigDim2() : ifVersion.getDim2();
- long elseVersionDim2 = (elseVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)elseVersion).getOrigDim2() : elseVersion.getDim2();
-
- if( ifVersionDim1 == elseVersionDim1 ){
- updatedDim1 = ifVersionDim1;
- }
- if( ifVersionDim2 == elseVersionDim2 ){
- updatedDim2 = ifVersionDim2;
- }
-
-
- //NOTE: nnz not propagated via validate, and hence, we conservatively assume that nnz have been changed.
- //if( ifVersion.getNnz() == elseVersion.getNnz() ){
- // updatedNnz = ifVersion.getNnz();
- //}
-
- // add reconsiled version (deep copy of ifVersion, cast as DataIdentifier)
- DataIdentifier recVersion = new DataIdentifier(ifVersion);
- recVersion.setDimensions(updatedDim1, updatedDim2);
- recVersion.setNnz(updatedNnz);
- recVars.addVariable(updatedVar, recVersion);
- }
- else {
- // CASE: defined only if branch
- DataIdentifier recVersion = null;
- if (ifVersion != null){
- // add reconciled version (deep copy of ifVersion, cast as DataIdentifier)
- recVersion = new DataIdentifier(ifVersion);
- recVars.addVariable(updatedVar, recVersion);
- }
- // CASE: defined only else branch
- else if (elseVersion != null){
- // add reconciled version (deep copy of elseVersion, cast as DataIdentifier)
- recVersion = new DataIdentifier(elseVersion);
- recVars.addVariable(updatedVar, recVersion);
- }
- // CASE: updated, but not in either if or else branch
- else {
- // add reconciled version (deep copy of elseVersion, cast as DataIdentifier)
- recVersion = new DataIdentifier(_updated.getVariable(updatedVar));
- recVars.addVariable(updatedVar, recVersion);
- }
-
-
- long updatedDim1 = -1, updatedDim2 = -1;
- long updatedNnz = -1;
-
- if( origVersion != null ) {
- long origVersionDim1 = (origVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)origVersion).getOrigDim1() : origVersion.getDim1();
- long recVersionDim1 = recVersion.getDim1(); //always DataIdentifier (see above)
- long origVersionDim2 = (origVersion instanceof IndexedIdentifier) ? ((IndexedIdentifier)origVersion).getOrigDim2() : origVersion.getDim2();
- long recVersionDim2 = recVersion.getDim2(); //always DataIdentifier (see above)
-
- if( origVersionDim1 == recVersionDim1 ){
- updatedDim1 = origVersionDim1;
- }
- if( origVersionDim2 == recVersionDim2 ){
- updatedDim2 = origVersionDim2;
- }
- //NOTE: nnz not propagated via validate, and hence, we conservatively assume that nnz have been changed.
- //if( origVersion.getNnz() == recVersion.getNnz() ){
- // updatedNnz = recVersion.getNnz();
- //}
- }
-
- recVersion.setDimensions(updatedDim1, updatedDim2);
- recVersion.setNnz(updatedNnz);
- }
- }
-
- // propagate updated variables
- VariableSet allIdVars = new VariableSet(recVars);
-
- _constVarsIn.putAll(constVars);
- _constVarsOut.putAll(recConstVars);
-
- return allIdVars;
- }
-
- @Override
- public VariableSet initializeforwardLV(VariableSet activeInPassed)
- throws LanguageException
- {
- IfStatement ifstmt = (IfStatement)_statements.get(0);
- if (_statements.size() > 1){
- LOG.error(ifstmt.printErrorLocation() + "IfStatementBlock should have only 1 statement (if statement)");
- throw new LanguageException(ifstmt.printErrorLocation() + "IfStatementBlock should have only 1 statement (if statement)");
- }
- _read = new VariableSet();
- _gen = new VariableSet();
- _kill = new VariableSet();
- _warnSet = new VariableSet();
-
- ///////////////////////////////////////////////////////////////////////
- // HANDLE PREDICATE
- ///////////////////////////////////////////////////////////////////////
- _read.addVariables(ifstmt.getConditionalPredicate().variablesRead());
- _updated.addVariables(ifstmt.getConditionalPredicate().variablesUpdated());
- _gen.addVariables(ifstmt.getConditionalPredicate().variablesRead());
-
- ///////////////////////////////////////////////////////////////////////
- // IF STATEMENT
- ///////////////////////////////////////////////////////////////////////
-
- // initialize forward for each statement block in if body
- VariableSet ifCurrent = new VariableSet(activeInPassed);
- VariableSet genIfBody = new VariableSet();
- VariableSet killIfBody = new VariableSet();
- VariableSet updatedIfBody = new VariableSet();
- VariableSet readIfBody = new VariableSet();
-
- for (StatementBlock sb : ifstmt.getIfBody())
- {
- ifCurrent = sb.initializeforwardLV(ifCurrent);
-
- // for each generated variable in this block, check variable not killed
- // (assigned value) in prior statement block in ifstmt blody
- for (String varName : sb._gen.getVariableNames()){
-
- // IF the variable is NOT set in the while loop PRIOR to this stmt block,
- // THEN needs to be generated
- if (!killIfBody.getVariableNames().contains(varName)){
- genIfBody.addVariable(varName, sb._gen.getVariable(varName));
- }
- }
-
- readIfBody.addVariables(sb._read);
- updatedIfBody.addVariables(sb._updated);
-
- // only add kill variables for statement blocks guaranteed to execute
- if (!(sb instanceof WhileStatementBlock) && !(sb instanceof ForStatementBlock) ){
- killIfBody.addVariables(sb._kill);
- }
- }
-
- ///////////////////////////////////////////////////////////////////////
- // ELSE STATEMENT
- ///////////////////////////////////////////////////////////////////////
-
- // initialize forward for each statement block in if body
- VariableSet elseCurrent = new VariableSet(activeInPassed);
- VariableSet genElseBody = new VariableSet();
- VariableSet killElseBody = new VariableSet();
- VariableSet updatedElseBody = new VariableSet();
- VariableSet readElseBody = new VariableSet();
-
- // initialize forward for each statement block in else body
- for (StatementBlock sb : ifstmt.getElseBody())
- {
- elseCurrent = sb.initializeforwardLV(elseCurrent);
-
- // for each generated variable in this block, check variable not killed
- // (assigned value) in prior statement block in ifstmt blody
- for (String varName : sb._gen.getVariableNames()){
-
- // IF the variable is NOT set in the while loop PRIOR to this stmt block,
- // THEN needs to be generated
- if (!killElseBody.getVariableNames().contains(varName)){
- genElseBody.addVariable(varName, sb._gen.getVariable(varName));
- }
- }
-
- readElseBody.addVariables(sb._read);
- updatedElseBody.addVariables(sb._updated);
-
- // only add kill variables for statement blocks guaranteed to execute
- if (!(sb instanceof WhileStatementBlock) && !(sb instanceof ForStatementBlock) ){
- killElseBody.addVariables(sb._kill);
- }
- }
-
- ///////////////////////////////////////////////////////////////////////
- // PERFORM RECONCILIATION
- ///////////////////////////////////////////////////////////////////////
-
- // "conservative" read -- union of read sets for if and else path
- _read.addVariables(readIfBody);
- _read.addVariables(readElseBody);
-
- // "conservative" update -- union of updated
- _updated.addVariables(updatedIfBody);
- _updated.addVariables(updatedElseBody);
-
- // "conservative" gen -- union of gen
- _gen.addVariables(genIfBody);
- _gen.addVariables(genElseBody);
-
- // "conservative" kill -- kill set is intersection of if-kill and else-kill
- for ( String varName : killIfBody.getVariableNames()){
- if (killElseBody.containsVariable(varName)){
- _kill.addVariable(varName, killIfBody.getVariable(varName));
- }
- }
-
- // set preliminary "warn" set -- variables that if used later may cause runtime error
- // if the loop is not executed
- // warnSet = (updated MINUS (updatedIfBody INTERSECT updatedElseBody)) MINUS current
- for (String varName : _updated.getVariableNames()){
- if (!((updatedIfBody.containsVariable(varName) && updatedElseBody.containsVariable(varName))
- || activeInPassed.containsVariable(varName))) {
- _warnSet.addVariable(varName, _updated.getVariable(varName));
- }
- }
-
-
- // set activeOut to (if body current UNION else body current) UNION updated
- _liveOut = new VariableSet();
- _liveOut.addVariables(ifCurrent);
- _liveOut.addVariables(elseCurrent);
- _liveOut.addVariables(_updated);
- return _liveOut;
- }
-
- @Override
- public VariableSet initializebackwardLV(VariableSet loPassed)
- throws LanguageException
- {
- IfStatement ifstmt = (IfStatement)_statements.get(0);
- if (_statements.size() > 1){
- LOG.error(ifstmt.printErrorLocation() + "IfStatementBlock should have only 1 statement (if statement)");
- throw new LanguageException(ifstmt.printErrorLocation() + "IfStatementBlock should have only 1 statement (if statement)");
- }
-
- VariableSet currentLiveOutIf = new VariableSet(loPassed);
- VariableSet currentLiveOutElse = new VariableSet(loPassed);
-
- int numBlocks = ifstmt.getIfBody().size();
- for (int i = numBlocks - 1; i >= 0; i--){
- currentLiveOutIf = ifstmt.getIfBody().get(i).analyze(currentLiveOutIf);
- }
-
- numBlocks = ifstmt.getElseBody().size();
- for (int i = numBlocks - 1; i >= 0; i--){
- currentLiveOutElse = ifstmt.getElseBody().get(i).analyze(currentLiveOutElse);
- }
-
- // Any variable defined in either if-body or else-body is available for later use
- VariableSet bothPathsLiveOut = new VariableSet();
- bothPathsLiveOut.addVariables(currentLiveOutIf);
- bothPathsLiveOut.addVariables(currentLiveOutElse);
-
- return bothPathsLiveOut;
-
- }
-
- public void setPredicateHops(Hop hops) {
- _predicateHops = hops;
- }
-
- public ArrayList<Hop> get_hops() throws HopsException{
-
- if (_hops != null && _hops.size() > 0){
- LOG.error(this.printBlockErrorLocation() + "error there should be no HOPs in IfStatementBlock");
- throw new HopsException(this.printBlockErrorLocation() + "error there should be no HOPs in IfStatementBlock");
- }
-
- return _hops;
- }
-
- public Hop getPredicateHops(){
- return _predicateHops;
- }
-
- public Lop get_predicateLops() {
- return _predicateLops;
- }
-
- public void set_predicateLops(Lop predicateLops) {
- _predicateLops = predicateLops;
- }
-
- public VariableSet analyze(VariableSet loPassed) throws LanguageException{
-
- VariableSet predVars = ((IfStatement)_statements.get(0)).getConditionalPredicate().variablesRead();
- predVars.addVariables(((IfStatement)_statements.get(0)).getConditionalPredicate().variablesUpdated());
-
- VariableSet candidateLO = new VariableSet(loPassed);
- VariableSet origLiveOut = new VariableSet(_liveOut);
-
- _liveOut = new VariableSet();
- for (String name : candidateLO.getVariableNames()){
- if (origLiveOut.containsVariable(name)){
- _liveOut.addVariable(name, candidateLO.getVariable(name));
- }
- }
-
- initializebackwardLV(_liveOut);
-
- // set final warnSet: remove variables NOT in live out
- VariableSet finalWarnSet = new VariableSet();
- for (String varName : _warnSet.getVariableNames()){
- if (_liveOut.containsVariable(varName)){
- finalWarnSet.addVariable(varName,_warnSet.getVariable(varName));
- }
- }
- _warnSet = finalWarnSet;
-
- // for now just print the warn set
- for (String varName : _warnSet.getVariableNames()){
- LOG.warn(_warnSet.getVariable(varName).printWarningLocation() + "Initialization of " + varName + " depends on if-else execution");
- }
-
- //data flow equation: liveout = gen \cup (liveout - kill)
- _liveIn = new VariableSet();
- _liveIn.addVariables(_liveOut);
- _liveIn.removeVariables(_kill);
- _liveIn.addVariables(_gen);
-
- VariableSet liveInReturn = new VariableSet();
- liveInReturn.addVariables(_liveIn);
- return liveInReturn;
- }
-
-
- /////////
- // materialized hops recompilation flags
- ////
-
- public void updatePredicateRecompilationFlag()
- throws HopsException
- {
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && Recompiler.requiresRecompilation(getPredicateHops());
- }
-
- public boolean requiresPredicateRecompilation()
- {
- return _requiresPredicateRecompile;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ImportStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ImportStatement.java b/src/main/java/com/ibm/bi/dml/parser/ImportStatement.java
deleted file mode 100644
index 87c9a4d..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ImportStatement.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class ImportStatement extends Statement
-{
-
- /*
- // indicates the imported list of function names
- private ArrayList<String> _importVarList;
-
- // Indicates if the "import all" option (import moudle (*)) is being used
- private boolean _importAll;
- */
-
- // Stores the namespace methods for
- private String _namespace;
-
- // Stores the filepath with the DML file location
- private String _filePath;
-
- // store "complete path" -- DML working dir + filepath
- private String _completePath;
-
-
- public ImportStatement(){
- _namespace = null;
- _filePath = null;
- _completePath = null;
-
- }
-
- public void setNamespace(String passed){
- _namespace = passed;
- }
-
- public String getNamespace(){
- return _namespace;
- }
-
- public String getFilePath() {
- return _filePath;
- }
-
- public void setFilePath(String filePath) {
- _filePath = filePath;
- }
-
- public String getCompletePath() {
- return _completePath;
- }
-
- public void setCompletePath(String filePath) {
- _completePath = filePath;
- }
-
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- LOG.error(this.printErrorLocation() + "rewriting for inlining not supported for ImportStatement");
- throw new LanguageException(this.printErrorLocation() + "rewriting for inlining not supported for ImportStatement");
- }
-
- public void initializeforwardLV(VariableSet activeIn){}
-
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public boolean controlStatement() {
- return false;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(Statement.SOURCE + "(");
- sb.append(this._filePath + ")");
-
- if (this._namespace != null){
- sb.append(" AS " + this._namespace);
- }
- sb.append(";");
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/IndexedIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/IndexedIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/IndexedIdentifier.java
deleted file mode 100644
index a23c663..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/IndexedIdentifier.java
+++ /dev/null
@@ -1,782 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-
-public class IndexedIdentifier extends DataIdentifier
-{
-
- // stores the expressions containing the ranges for the
- private Expression _rowLowerBound = null, _rowUpperBound = null, _colLowerBound = null, _colUpperBound = null;
-
- // stores whether row / col indices have same value (thus selecting either (1 X n) row-vector OR (n X 1) col-vector)
- private boolean _rowLowerEqualsUpper = false, _colLowerEqualsUpper = false;
-
- // for IndexedIdentifier, dim1 and dim2 will ultimately be the dimensions of the indexed region and NOT the dims of what is being indexed
- // E.g., for A[1:10,1:10], where A = Rand (rows = 20, cols = 20), dim1 = 10, dim2 = 10, origDim1 = 20, origDim2 = 20
-
- // stores the dimensions of Identifier prior to indexing
- private long _origDim1, _origDim2;
-
- public boolean getRowLowerEqualsUpper(){
- return _rowLowerEqualsUpper;
- }
-
- public boolean getColLowerEqualsUpper() {
- return _colLowerEqualsUpper;
- }
-
- public void setRowLowerEqualsUpper(boolean passed){
- _rowLowerEqualsUpper = passed;
- }
-
- public void setColLowerEqualsUpper(boolean passed) {
- _colLowerEqualsUpper = passed;
- }
-
- public IndexedIdentifier(String name, boolean passedRows, boolean passedCols){
- super(name);
- _rowLowerBound = null;
- _rowUpperBound = null;
- _colLowerBound = null;
- _colUpperBound = null;
-
- _rowLowerEqualsUpper = passedRows;
- _colLowerEqualsUpper = passedCols;
-
- _origDim1 = -1L;
- _origDim2 = -1L;
- }
-
- /**
- *
- * @param ids
- * @param currConstVars
- * @param conditional
- * @return
- * @throws LanguageException
- */
- public IndexPair calculateIndexedDimensions(HashMap<String,DataIdentifier> ids, HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
- throws LanguageException
- {
- // stores the updated row / col dimension info
- long updatedRowDim = -1, updatedColDim = -1;
-
- boolean isConst_rowLowerBound = false;
- boolean isConst_rowUpperBound = false;
- boolean isConst_colLowerBound = false;
- boolean isConst_colUpperBound = false;
-
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // STEP 1 : perform constant propagation for index boundaries
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // PROCESS ROW LOWER BOUND
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- // process row lower bound
- if (_rowLowerBound instanceof ConstIdentifier && ( _rowLowerBound instanceof IntIdentifier || _rowLowerBound instanceof DoubleIdentifier )){
-
- Long rowLB_1_1 = -1L;
- if (_rowLowerBound instanceof IntIdentifier)
- rowLB_1_1 = ((IntIdentifier)_rowLowerBound).getValue();
- else
- rowLB_1_1 = UtilFunctions.toLong(((DoubleIdentifier)_rowLowerBound).getValue());
-
- if (rowLB_1_1 < 1){
- raiseValidateError("lower-bound row index " + rowLB_1_1 + " initialized to out of bounds value. Value must be >= 1", conditional);
- }
- if ((this.getOrigDim1() > 0) && (rowLB_1_1 > this.getOrigDim1())) {
- raiseValidateError("lower-bound row index " + rowLB_1_1 + " initialized to out of bounds value. Rows in " + this.getName() + ": " + this.getOrigDim1(), conditional);
- }
- // valid lower row bound value
- isConst_rowLowerBound = true;
- }
-
- else if (_rowLowerBound instanceof ConstIdentifier) {
- raiseValidateError("assign lower-bound row index for Indexed Identifier " + this.toString() + " the non-numeric value " + _rowLowerBound.toString(), conditional);
- }
-
- // perform constant propogation for row lower bound
- else if (_rowLowerBound != null && _rowLowerBound instanceof DataIdentifier && !(_rowLowerBound instanceof IndexedIdentifier)) {
- String identifierName = ((DataIdentifier)_rowLowerBound).getName();
-
- // CASE: rowLowerBound is a constant DataIdentifier
- if (currConstVars.containsKey(identifierName)){
- ConstIdentifier constValue = currConstVars.get(identifierName);
-
- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- LOG.info(this.printInfoLocation() + "attempted to assign lower row bound for " + this.toString() + "the non-numeric value " + constValue.getOutput().toString() + " assigned to " + identifierName + ". May cause runtime exception ");
-
- else {
-
- // test constant propogation
- long tempRowLB = -1L;
- boolean validRowLB = true;
- if (constValue instanceof IntIdentifier)
- tempRowLB = ((IntIdentifier)constValue).getValue();
- else
- tempRowLB = UtilFunctions.toLong(((DoubleIdentifier)constValue).getValue());
-
- if (tempRowLB < 1){
- LOG.info(this.printInfoLocation() + "lower-bound row index " + identifierName + " initialized to " + tempRowLB + " May cause runtime exception (runtime value must be >= 1)");
- validRowLB = false;
- }
- if (this.getOrigDim1() > 0 && tempRowLB > this.getOrigDim1()){
- LOG.info(this.printInfoLocation() + "lower-bound row index " + identifierName
- + " initialized to " + tempRowLB + " May cause runtime exception (Rows in "
- + this.getName() + ": " + this.getOrigDim1() +")");
- validRowLB = false;
- }
-
- if (validRowLB) {
- if (constValue instanceof IntIdentifier){
- _rowLowerBound = new IntIdentifier((IntIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(), constValue.getBeginColumn(),
- constValue.getEndLine(), constValue.getEndColumn());
-
- }
- else{
- _rowLowerBound = new DoubleIdentifier((DoubleIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(), constValue.getBeginColumn(),
- constValue.getEndLine(), constValue.getEndColumn());
- }
- isConst_rowLowerBound = true;
- }
- } // end else -- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- } // end if (currConstVars.containsKey(identifierName))
- } // end constant propogation for row LB
-
- // check 1 < indexed row lower-bound < rows in IndexedIdentifier
- // (assuming row dims available for upper bound)
- Long rowLB_1 = -1L;
- if (isConst_rowLowerBound) {
-
- if (_rowLowerBound instanceof IntIdentifier)
- rowLB_1 = ((IntIdentifier)_rowLowerBound).getValue();
- else
- rowLB_1 = UtilFunctions.toLong(((DoubleIdentifier)_rowLowerBound).getValue());
- }
-
-
-
- ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // PROCESS ROW UPPER BOUND
- ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- if (_rowUpperBound instanceof ConstIdentifier && ( _rowUpperBound instanceof IntIdentifier || _rowUpperBound instanceof DoubleIdentifier )){
-
- Long rowUB_1_1 = -1L;
- if (_rowUpperBound instanceof IntIdentifier)
- rowUB_1_1 = ((IntIdentifier)_rowUpperBound).getValue();
- else
- rowUB_1_1 = UtilFunctions.toLong(((DoubleIdentifier)_rowUpperBound).getValue());
-
- if (rowUB_1_1 < 1){
- raiseValidateError("upper-bound row index " + rowUB_1_1 + " out of bounds value. Value must be >= 1", conditional);
- }
- if ((this.getOrigDim1() > 0) && (rowUB_1_1 > this.getOrigDim1())) {
- raiseValidateError("upper-bound row index " + rowUB_1_1 + " out of bounds value. Rows in " + this.getName() + ": " + this.getOrigDim1(), conditional);
- }
- if (isConst_rowLowerBound && rowUB_1_1 < rowLB_1){
- raiseValidateError("upper-bound row index " + rowUB_1_1 + " greater than lower-bound row index " + rowLB_1, conditional);
- }
- isConst_rowUpperBound = true;
- }
- else if (_rowUpperBound instanceof ConstIdentifier){
- raiseValidateError("assign upper-bound row index for " + this.toString() + " the non-numeric value " + _rowUpperBound.toString(), conditional);
- }
-
- // perform constant propogation for upper row index
- else if (_rowUpperBound != null && _rowUpperBound instanceof DataIdentifier && !(_rowUpperBound instanceof IndexedIdentifier)) {
- String identifierName = ((DataIdentifier)_rowUpperBound).getName();
-
- if (currConstVars.containsKey(identifierName)){
- ConstIdentifier constValue = currConstVars.get(identifierName);
-
- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- LOG.info(this.printInfoLocation() + "attempted to assign upper row bound for " + this.toString() + "the non-numeric value " + constValue.getOutput().toString() + " assigned to " + identifierName + ". May cause runtime exception ");
-
- else {
- // test constant propogation
- long tempRowUB = -1L;
- boolean validRowUB = true;
- if (constValue instanceof IntIdentifier)
- tempRowUB = ((IntIdentifier)constValue).getValue();
- else
- tempRowUB = UtilFunctions.toLong(((DoubleIdentifier)constValue).getValue());
-
- if (tempRowUB < 1){
- LOG.info(this.printInfoLocation() + "upper-bound row index " + identifierName + " initialized to " + tempRowUB + " May cause runtime exception (runtime value must be >= 1)");
- validRowUB = false;
- }
- if (this.getOrigDim1() > 0 && tempRowUB > this.getOrigDim1()){
- LOG.info(this.printInfoLocation() + "upper-bound row index " + identifierName + " initialized to " + tempRowUB + " May cause runtime exception (Rows in " + this.getName() + ": " + this.getOrigDim1() +")");
- validRowUB = false;
- }
- if (isConst_rowLowerBound && tempRowUB < rowLB_1){
- LOG.info(this.printInfoLocation() + "upper-bound row index "
- + identifierName + " initialized to " + tempRowUB
- + ", which is greater than lower-bound row index value "
- + rowLB_1 + " May cause runtime exception");
- validRowUB = false;
- }
-
- if (validRowUB) {
- if (constValue instanceof IntIdentifier){
- _rowUpperBound = new IntIdentifier((IntIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- else{
- _rowUpperBound = new DoubleIdentifier((DoubleIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- isConst_rowUpperBound = true;
- }
- } // end else -- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- } // end if (currConstVars.containsKey(identifierName)){
-
- } // end constant propagation for row UB
-
-
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // PROCESS COLUMN LOWER BOUND
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- // process column lower bound
- if (_colLowerBound instanceof ConstIdentifier && ( _colLowerBound instanceof IntIdentifier || _colLowerBound instanceof DoubleIdentifier )){
-
- Long colLB_1_1 = -1L;
- if (_colLowerBound instanceof IntIdentifier)
- colLB_1_1 = ((IntIdentifier)_colLowerBound).getValue();
- else
- colLB_1_1 = UtilFunctions.toLong(((DoubleIdentifier)_colLowerBound).getValue());
-
- if (colLB_1_1 < 1){
- raiseValidateError("lower-bound column index " + colLB_1_1 + " initialized to out of bounds value. Value must be >= 1", conditional);
- }
- if ((this.getOrigDim2() > 0) && (colLB_1_1 > this.getOrigDim2())){
- raiseValidateError("lower-bound column index " + colLB_1_1 + " initialized to out of bounds value. Columns in " + this.getName() + ": " + this.getOrigDim2(), conditional);
- }
- // valid lower row bound value
- isConst_colLowerBound = true;
- }
-
- else if (_colLowerBound instanceof ConstIdentifier) {
- raiseValidateError("assign lower-bound column index for Indexed Identifier " + this.toString() + " the non-numeric value " + _colLowerBound.toString(), conditional);
- }
-
- // perform constant propogation for column lower bound
- else if (_colLowerBound != null && _colLowerBound instanceof DataIdentifier && !(_colLowerBound instanceof IndexedIdentifier)) {
- String identifierName = ((DataIdentifier)_colLowerBound).getName();
- if (currConstVars.containsKey(identifierName)){
- ConstIdentifier constValue = currConstVars.get(identifierName);
-
- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- LOG.info(this.printInfoLocation() + "attempted to assign lower column bound for " + this.toString() + "the non-numeric value " + constValue.getOutput().toString() + " assigned to " + identifierName + ". May cause runtime exception ");
-
- else {
-
- // test constant propogation
- long tempColLB = -1L;
- boolean validColLB = true;
- if (constValue instanceof IntIdentifier)
- tempColLB = ((IntIdentifier)constValue).getValue();
- else
- tempColLB = UtilFunctions.toLong(((DoubleIdentifier)constValue).getValue());
-
- if (tempColLB < 1){
- LOG.info(this.printInfoLocation() + "lower-bound column index " + identifierName + " initialized to " + tempColLB + " May cause runtime exception (runtime value must be >= 1)");
- validColLB = false;
- }
- if (this.getOrigDim2() > 0 && tempColLB > this.getOrigDim2()){
- LOG.info(this.printInfoLocation() + "lower-bound column index "
- + identifierName + " initialized to " + tempColLB
- + " May cause runtime exception (Columns in " + this.getName()
- + ": " + this.getOrigDim2() +")");
- validColLB = false;
- }
-
- if (validColLB) {
- if (constValue instanceof IntIdentifier){
- _colLowerBound = new IntIdentifier((IntIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- else{
- _colLowerBound = new DoubleIdentifier((DoubleIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- isConst_colLowerBound = true;
- }
- } // end else -- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- }
- } // end constant propogation for column LB
-
- // check 1 < indexed column lower-bound < columns in IndexedIdentifier
- // (assuming column dims available for upper bound)
- Long colLB_1 = -1L;
- if (isConst_colLowerBound) {
-
- if (_colLowerBound instanceof IntIdentifier)
- colLB_1 = ((IntIdentifier)_colLowerBound).getValue();
- else
- colLB_1 = UtilFunctions.toLong(((DoubleIdentifier)_colLowerBound).getValue());
- }
-
-
-
- ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // PROCESS ROW UPPER BOUND
- ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- if (_colUpperBound instanceof ConstIdentifier && ( _colUpperBound instanceof IntIdentifier || _colUpperBound instanceof DoubleIdentifier )){
-
- Long colUB_1_1 = -1L;
- if (_colUpperBound instanceof IntIdentifier)
- colUB_1_1 = ((IntIdentifier)_colUpperBound).getValue();
- else
- colUB_1_1 = UtilFunctions.toLong(((DoubleIdentifier)_colUpperBound).getValue());
-
- if (colUB_1_1 < 1){
- raiseValidateError("upper-bound column index " + colUB_1_1 + " out of bounds value. Value must be >= 1", conditional);
- }
- if ((this.getOrigDim2() > 0) && (colUB_1_1 > this.getOrigDim2())) {
- raiseValidateError("upper-bound column index " + colUB_1_1 + " out of bounds value. Columns in " + this.getName() + ": " + this.getOrigDim2(), conditional);
- }
- if (isConst_rowLowerBound && colUB_1_1 < colLB_1) {
- raiseValidateError("upper-bound column index " + colUB_1_1 + " greater than lower-bound row index " + colLB_1, conditional);
- }
-
- isConst_colUpperBound = true;
- }
- else if (_colUpperBound instanceof ConstIdentifier){
- raiseValidateError("assign upper-bound column index for " + this.toString() + " the non-numeric value " + _colUpperBound.toString(), conditional);
- }
-
- // perform constant propogation for upper row index
- else if (_colUpperBound != null && _colUpperBound instanceof DataIdentifier
- && !(_colUpperBound instanceof IndexedIdentifier)) {
- String identifierName = ((DataIdentifier)_colUpperBound).getName();
-
-
- if (currConstVars.containsKey(identifierName)){
- ConstIdentifier constValue = currConstVars.get(identifierName);
-
- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- LOG.info(this.printInfoLocation() + "attempted to assign upper column bound for "
- + this.toString() + "the non-numeric value " + constValue.getOutput().toString()
- + " assigned to " + identifierName + ". May cause runtime exception ");
-
- else {
- // test constant propogation
- long tempColUB = -1L;
- boolean validColUB = true;
- if (constValue instanceof IntIdentifier)
- tempColUB = ((IntIdentifier)constValue).getValue();
- else
- tempColUB = UtilFunctions.toLong(((DoubleIdentifier)constValue).getValue());
-
- if (tempColUB < 1){
- LOG.info(this.printInfoLocation() + "upper-bound column index "
- + identifierName + " initialized to " + tempColUB
- + " May cause runtime exception (runtime value must be >= 1)");
- validColUB = false;
- }
- if (this.getOrigDim2() > 0 && tempColUB > this.getOrigDim2()){
- LOG.info(this.printInfoLocation() + "upper-bound column index "
- + identifierName + " initialized to " + tempColUB
- + " May cause runtime exception (Columns in " + this.getName()
- + ": " + this.getOrigDim2() + ")");
- validColUB = false;
- }
- if (isConst_colLowerBound && tempColUB < colLB_1){
- LOG.info(this.printInfoLocation() + "upper-bound column index "
- + identifierName + " initialized to " + tempColUB
- + ", which is greater than lower-bound column index value " + colLB_1
- + " May cause runtime exception");
- validColUB = false;
- }
-
- if (validColUB) {
- if (constValue instanceof IntIdentifier){
- _colUpperBound = new IntIdentifier((IntIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- else{
- _colUpperBound = new DoubleIdentifier((DoubleIdentifier)constValue,
- constValue.getFilename(), constValue.getBeginLine(),
- constValue.getBeginColumn(), constValue.getEndLine(),
- constValue.getEndColumn());
- }
- isConst_colUpperBound = true;
- }
- } // end else -- if (!(constValue instanceof IntIdentifier || constValue instanceof DoubleIdentifier ))
- }
- } // end constant propagation for column UB
-
- ///////////////////////////////////////////////////////////////////////
- // STEP 2: update row dimensions
- ///////////////////////////////////////////////////////////////////////
-
- // CASE: lower == upper --> updated row dim = 1
- if (_rowLowerEqualsUpper)
- updatedRowDim = 1;
-
- // CASE: (lower == null && upper == null) --> updated row dim = (rows of input)
- else if (_rowLowerBound == null && _rowUpperBound == null){
- updatedRowDim = this.getOrigDim1();
- }
-
- // CASE: (lower == null) && (upper == constant) --> updated row dim = (constant)
- else if (_rowLowerBound == null && isConst_rowUpperBound) {
- if (_rowUpperBound instanceof IntIdentifier)
- updatedRowDim = ((IntIdentifier)_rowUpperBound).getValue();
- else if (_rowUpperBound instanceof DoubleIdentifier)
- updatedRowDim = UtilFunctions.toLong(((DoubleIdentifier)_rowUpperBound).getValue());
- }
-
- // CASE: (lower == constant) && (upper == null) && (dimIndex > 0) --> rowCount - lower bound + 1
- else if (isConst_rowLowerBound && _rowUpperBound == null && this.getOrigDim1() > 0) {
- long rowCount = this.getOrigDim1();
- if (_rowLowerBound instanceof IntIdentifier)
- updatedRowDim = rowCount - ((IntIdentifier)_rowLowerBound).getValue() + 1;
- else if (_rowLowerBound instanceof DoubleIdentifier)
- updatedRowDim = UtilFunctions.toLong(rowCount - ((DoubleIdentifier)_rowLowerBound).getValue() + 1);
- }
- // CASE: (lower == constant) && (upper == constant) --> upper bound - lower bound + 1
- else if (isConst_rowLowerBound && isConst_rowUpperBound) {
- if (_rowLowerBound instanceof IntIdentifier && _rowUpperBound instanceof IntIdentifier)
- updatedRowDim = ((IntIdentifier)_rowUpperBound).getValue() - ((IntIdentifier)_rowLowerBound).getValue() + 1;
-
- else if (_rowLowerBound instanceof DoubleIdentifier && _rowUpperBound instanceof DoubleIdentifier)
- updatedRowDim = UtilFunctions.toLong( ((DoubleIdentifier)_rowUpperBound).getValue() - ((DoubleIdentifier)_rowLowerBound).getValue() + 1);
-
- else if (_rowLowerBound instanceof IntIdentifier && _rowUpperBound instanceof DoubleIdentifier)
- updatedRowDim = UtilFunctions.toLong( ((DoubleIdentifier)_rowUpperBound).getValue() - ((IntIdentifier)_rowLowerBound).getValue() + 1);
-
- else if (_rowLowerBound instanceof DoubleIdentifier && _rowUpperBound instanceof IntIdentifier)
- updatedRowDim = UtilFunctions.toLong( ((IntIdentifier)_rowUpperBound).getValue() - ((DoubleIdentifier)_rowLowerBound).getValue() + 1);
-
- }
-
- // CASE: row dimension is unknown --> assign -1
- else{
- updatedRowDim = -1;
- }
-
-
- //////////////////////////////////////////////////////////////////////
- // STEP 3: update column dimensions
- ///////////////////////////////////////////////////////////////////////
-
- // CASE: lower == upper --> updated col dim = 1
- if (_colLowerEqualsUpper)
- updatedColDim = 1;
-
- // CASE: (lower == null && upper == null) --> updated col dim = (cols of input)
- else if (_colLowerBound == null && _colUpperBound == null){
- updatedColDim = this.getOrigDim2();
- }
- // CASE: (lower == null) && (upper == constant) --> updated col dim = (constant)
- else if (_colLowerBound == null && isConst_colUpperBound) {
- if (_colUpperBound instanceof IntIdentifier)
- updatedColDim = ((IntIdentifier)_colUpperBound).getValue();
- else if (_colUpperBound instanceof DoubleIdentifier)
- updatedColDim = UtilFunctions.toLong(((DoubleIdentifier)_colUpperBound).getValue());
- }
-
- // CASE: (lower == constant) && (upper == null) && (dimIndex > 0) --> colCount - lower bound + 1
- else if (isConst_colLowerBound && _colUpperBound == null && this.getOrigDim2() > 0) {
- long colCount = this.getOrigDim2();
- if (_colLowerBound instanceof IntIdentifier)
- updatedColDim = colCount - ((IntIdentifier)_colLowerBound).getValue() + 1;
- else if (_colLowerBound instanceof DoubleIdentifier)
- updatedColDim = UtilFunctions.toLong(colCount - ((DoubleIdentifier)_colLowerBound).getValue() + 1);
- }
-
- // CASE: (lower == constant) && (upper == constant) --> upper bound - lower bound + 1
- else if (isConst_colLowerBound && isConst_colUpperBound) {
- if (_colLowerBound instanceof IntIdentifier && _colUpperBound instanceof IntIdentifier)
- updatedColDim = ((IntIdentifier)_colUpperBound).getValue() - ((IntIdentifier)_colLowerBound).getValue() + 1;
-
- else if (_colLowerBound instanceof DoubleIdentifier && _colUpperBound instanceof DoubleIdentifier)
- updatedColDim = UtilFunctions.toLong( ((DoubleIdentifier)_colUpperBound).getValue() - ((DoubleIdentifier)_colLowerBound).getValue() + 1);
-
- else if (_colLowerBound instanceof IntIdentifier && _colUpperBound instanceof DoubleIdentifier)
- updatedColDim = UtilFunctions.toLong( ((DoubleIdentifier)_colUpperBound).getValue() - ((IntIdentifier)_colLowerBound).getValue() + 1);
-
- else if (_colLowerBound instanceof DoubleIdentifier && _colUpperBound instanceof IntIdentifier)
- updatedColDim = UtilFunctions.toLong( ((IntIdentifier)_colUpperBound).getValue() - ((DoubleIdentifier)_colLowerBound).getValue() + 1);
-
- }
-
- // CASE: column dimension is unknown --> assign -1
- else{
- updatedColDim = -1;
- }
-
- return new IndexPair(updatedRowDim, updatedColDim);
-
- }
-
- public void setOriginalDimensions(long passedDim1, long passedDim2){
- this._origDim1 = passedDim1;
- this._origDim2 = passedDim2;
- }
-
- public long getOrigDim1() { return this._origDim1; }
- public long getOrigDim2() { return this._origDim2; }
-
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- IndexedIdentifier newIndexedIdentifier = new IndexedIdentifier(this.getName(), this._rowLowerEqualsUpper, this._colLowerEqualsUpper);
-
- newIndexedIdentifier.setFilename(getFilename());
- newIndexedIdentifier.setBeginLine(this.getBeginLine());
- newIndexedIdentifier.setBeginColumn(this.getBeginColumn());
- newIndexedIdentifier.setEndLine(this.getEndLine());
- newIndexedIdentifier.setEndColumn(this.getEndColumn());
-
- // set dimensionality information and other Identifier specific properties for new IndexedIdentifier
- newIndexedIdentifier.setProperties(this);
- newIndexedIdentifier.setOriginalDimensions(this._origDim1, this._origDim2);
-
- // set remaining properties (specific to DataIdentifier)
- newIndexedIdentifier._kind = Kind.Data;
- newIndexedIdentifier._name = prefix + this._name;
- newIndexedIdentifier._valueTypeString = this.getValueType().toString();
- newIndexedIdentifier._defaultValue = this._defaultValue;
-
- // creates rewritten expression (deep copy)
- newIndexedIdentifier._rowLowerBound = (_rowLowerBound == null) ? null : _rowLowerBound.rewriteExpression(prefix);
- newIndexedIdentifier._rowUpperBound = (_rowUpperBound == null) ? null : _rowUpperBound.rewriteExpression(prefix);
- newIndexedIdentifier._colLowerBound = (_colLowerBound == null) ? null : _colLowerBound.rewriteExpression(prefix);
- newIndexedIdentifier._colUpperBound = (_colUpperBound == null) ? null : _colUpperBound.rewriteExpression(prefix);
-
- return newIndexedIdentifier;
- }
-
- public void setIndices(ArrayList<ArrayList<Expression>> passed) throws DMLParseException {
- if (passed.size() != 2){
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "matrix indices must be specified for 2 dimensions");
- }
- ArrayList<Expression> rowIndices = passed.get(0);
- ArrayList<Expression> colIndices = passed.get(1);
-
- // case: both upper and lower are defined
- if (rowIndices.size() == 2){
- _rowLowerBound = rowIndices.get(0);
- _rowUpperBound = rowIndices.get(1);
- }
- // case: only one index is defined --> thus lower = upper
- else if (rowIndices.size() == 1){
- _rowLowerBound = rowIndices.get(0);
- _rowUpperBound = rowIndices.get(0);
- _rowLowerEqualsUpper = true;
- }
- else {
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "row indices are length " + rowIndices.size() + " -- should be either 1 or 2");
- }
-
- // case: both upper and lower are defined
- if (colIndices.size() == 2){
- _colLowerBound = colIndices.get(0);
- _colUpperBound = colIndices.get(1);
- }
- // case: only one index is defined --> thus lower = upper
- else if (colIndices.size() == 1){
- _colLowerBound = colIndices.get(0);
- _colUpperBound = colIndices.get(0);
- _colLowerEqualsUpper = true;
- }
- else {
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "column indices are length " + + colIndices.size() + " -- should be either 1 or 2");
- }
-
- if (_rowLowerBound instanceof FunctionCallIdentifier || _rowUpperBound instanceof FunctionCallIdentifier
- || _colLowerBound instanceof FunctionCallIdentifier || _colUpperBound instanceof FunctionCallIdentifier){
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "UDF functions not supported for row or column indices");
- }
-
- }
-
- public Expression getRowLowerBound(){ return this._rowLowerBound; }
- public Expression getRowUpperBound(){ return this._rowUpperBound; }
- public Expression getColLowerBound(){ return this._colLowerBound; }
- public Expression getColUpperBound(){ return this._colUpperBound; }
-
- public void setRowLowerBound(Expression passed){ this._rowLowerBound = passed; }
- public void setRowUpperBound(Expression passed){ this._rowUpperBound = passed; }
- public void setColLowerBound(Expression passed){ this._colLowerBound = passed; }
- public void setColUpperBound(Expression passed){ this._colUpperBound = passed; }
-
- @Override
- public String toString() {
- String retVal = getName();
- if (_rowLowerBound != null || _rowUpperBound != null || _colLowerBound != null || _colUpperBound != null){
- retVal += "[";
-
- if (_rowLowerBound != null && _rowUpperBound != null){
- if (_rowLowerBound.toString().equals(_rowUpperBound.toString()))
- retVal += _rowLowerBound.toString();
- else
- retVal += _rowLowerBound.toString() + ":" + _rowUpperBound.toString();
- }
- else {
- if (_rowLowerBound != null || _rowUpperBound != null){
- if (_rowLowerBound != null)
- retVal += _rowLowerBound.toString();
-
- retVal += ":";
-
- if (_rowUpperBound != null)
- retVal += _rowUpperBound.toString();
- }
- }
-
- retVal += ",";
-
- if (_colLowerBound != null && _colUpperBound != null){
- if (_colLowerBound.toString().equals(_colUpperBound.toString()))
- retVal += _colLowerBound.toString();
- else
- retVal += _colLowerBound.toString() + ":" + _colUpperBound.toString();
- }
- else {
- if (_colLowerBound != null || _colUpperBound != null) {
-
- if (_colLowerBound != null)
- retVal += _colLowerBound.toString();
-
- retVal += ":";
-
- if (_colUpperBound != null)
- retVal += _colUpperBound.toString();
- }
- }
-
-
- retVal += "]";
- }
- return retVal;
- }
-
- @Override
- // handles case when IndexedIdentifier is on RHS for assignment
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- // add variable being indexed to read set
- result.addVariable(this.getName(), this);
-
- // add variables for indexing expressions
- if (_rowLowerBound != null)
- result.addVariables(_rowLowerBound.variablesRead());
- if (_rowUpperBound != null)
- result.addVariables(_rowUpperBound.variablesRead());
- if (_colLowerBound != null)
- result.addVariables(_colLowerBound.variablesRead());
- if (_colUpperBound != null)
- result.addVariables(_colUpperBound.variablesRead());
-
- return result;
- }
-
- public void setAllPositions (String filename, int blp, int bcp, ArrayList<ArrayList<Expression>> exprListList){
- setFilename(filename);
- setBeginLine(blp);
- setBeginColumn(bcp);
-
- Expression rightMostIndexExpr = null;
- if (exprListList != null){
- for(ArrayList<Expression> exprList : exprListList){
- if (exprList != null){
- for (Expression expr : exprList){
- if (expr != null) {
- rightMostIndexExpr = expr;
- }
- }
- }
- }
- }
-
- if (rightMostIndexExpr != null){
- setEndLine(rightMostIndexExpr.getEndLine());
- setEndColumn(rightMostIndexExpr.getEndColumn());
- }
- else {
- setEndLine(blp);
- setEndColumn(bcp);
- }
- }
-
-
- public void setProperties(Identifier i){
- _dataType = i.getDataType();
- _valueType = i.getValueType();
- _dim1 = i.getDim1();
- _dim2 = i.getDim2();
- _rows_in_block = i.getRowsInBlock();
- _columns_in_block = i.getColumnsInBlock();
- _nnz = i.getNnz();
- _formatType = i.getFormatType();
-
- if (i instanceof IndexedIdentifier){
- _origDim1 = ((IndexedIdentifier)i).getOrigDim1();
- _origDim2 = ((IndexedIdentifier)i).getOrigDim2();
- }
- else{
- _origDim1 = i.getDim1();
- _origDim2 = i.getDim2();
- }
- }
-
- @Override
- public boolean multipleReturns() {
- return false;
- }
-
-} // end class
-
-class IndexPair {
-
- public long _row, _col;
-
- public IndexPair (long row, long col){
- _row = row;
- _col = col;
- }
-} // end class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/IntIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/IntIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/IntIdentifier.java
deleted file mode 100644
index cb5446c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/IntIdentifier.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class IntIdentifier extends ConstIdentifier
-{
-
- private long _val;
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- return this;
- }
-
- public IntIdentifier(long val, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = val;
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.INT);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public IntIdentifier(IntIdentifier i, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = i.getValue();
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.INT);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- // Used only by the parser for unary operation
- public void multiplyByMinusOne() {
- _val = -1 * _val;
- }
-
- public long getValue(){
- return _val;
- }
-
- public String toString(){
- return Long.toString(_val);
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public long getLongValue(){
- return getValue();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/IterablePredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/IterablePredicate.java b/src/main/java/com/ibm/bi/dml/parser/IterablePredicate.java
deleted file mode 100644
index 0fab6db..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/IterablePredicate.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.lops.Lop;
-
-
-public class IterablePredicate extends Expression
-{
-
- private DataIdentifier _iterVar; // variable being iterated over
- private Expression _fromExpr;
- private Expression _toExpr;
- private Expression _incrementExpr;
- private HashMap<String,String> _parforParams;
-
-
- public IterablePredicate(DataIdentifier iterVar, Expression fromExpr, Expression toExpr,
- Expression incrementExpr, HashMap<String,String> parForParamValues,
- String filename, int blp, int bcp, int elp, int ecp)
- {
- _iterVar = iterVar;
- _fromExpr = fromExpr;
- _toExpr = toExpr;
- _incrementExpr = incrementExpr;
-
- _parforParams = parForParamValues;
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append( "(" );
- sb.append( _iterVar.getName() );
- sb.append(" in seq(");
- sb.append(_fromExpr.toString());
- sb.append(",");
- sb.append(_toExpr.toString());
- sb.append(",");
- sb.append(_incrementExpr.toString());
- sb.append( ")" );
- if (_parforParams != null && _parforParams.size() > 0){
- for (String key : _parforParams.keySet())
- {
- sb.append( "," );
- sb.append( key );
- sb.append( "=" );
- sb.append( _parforParams.get(key) );
- }
- }
- sb.append( ")" );
- return sb.toString();
- }
-
-
- public VariableSet variablesRead()
- {
- VariableSet result = new VariableSet();
- result.addVariables( _fromExpr.variablesRead() );
- result.addVariables( _toExpr.variablesRead() );
- result.addVariables( _incrementExpr.variablesRead() );
-
- return result;
- }
-
-
- public VariableSet variablesUpdated()
- {
- VariableSet result = new VariableSet();
- result.addVariable(_iterVar.getName(), _iterVar);
-
- return result;
- }
-
- @Override
- public Expression rewriteExpression(String prefix) throws LanguageException {
- //DataIdentifier newIterVar = (DataIdentifier)_iterVar.rewriteExpression(prefix);
- //return new IterablePredicate(newIterVar, _from, _to, _increment);
- LOG.error(this.printErrorLocation() + "rewriteExpression not supported for IterablePredicate");
- throw new LanguageException(this.printErrorLocation() + "rewriteExpression not supported for IterablePredicate");
-
- }
-
- @Override
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
-
- //recursive validate
- if (_iterVar instanceof FunctionCallIdentifier
- || _fromExpr instanceof FunctionCallIdentifier
- || _toExpr instanceof FunctionCallIdentifier
- || _incrementExpr instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported for iterable predicates",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
-
-
- //1) VALIDATE ITERATION VARIABLE (index)
- // check the variable has either 1) not been defined already OR 2) defined as integer scalar
- if (ids.containsKey(_iterVar.getName())){
- DataIdentifier otherDI = ids.get(_iterVar.getName());
- if( otherDI.getDataType() != DataType.SCALAR || otherDI.getValueType() != ValueType.INT ){
- raiseValidateError("iterable predicate in for loop '" + _iterVar.getName() + "' must be a scalar integer", conditional);
- }
- }
-
- // set the values for DataIdentifer iterable variable
- _iterVar.setIntProperties();
-
- // add the iterVar to the variable set
- ids.put(_iterVar.getName(), _iterVar);
-
-
- //2) VALIDATE FOR PREDICATE in (from, to, increment)
- //recursively validate the individual expression
- _fromExpr.validateExpression(ids, constVars, conditional);
- _toExpr.validateExpression(ids, constVars, conditional);
- _incrementExpr.validateExpression(ids, constVars, conditional);
-
- //check for scalar expression output
- checkNumericScalarOutput( _fromExpr );
- checkNumericScalarOutput( _toExpr );
- checkNumericScalarOutput( _incrementExpr );
- }
-
- public DataIdentifier getIterVar() {
- return _iterVar;
- }
-
- public void setIterVar(DataIdentifier iterVar) {
- _iterVar = iterVar;
- }
-
- public Expression getFromExpr() {
- return _fromExpr;
- }
-
- public void setFromExpr(Expression from) {
- _fromExpr = from;
- }
-
- public Expression getToExpr() {
- return _toExpr;
- }
-
- public void setToExpr(Expression to) {
- _toExpr = to;
- }
-
- public Expression getIncrementExpr() {
- return _incrementExpr;
- }
-
- public void setIncrementExpr(Expression increment) {
- _incrementExpr = increment;
- }
-
- public HashMap<String,String> getParForParams(){
- return _parforParams;
- }
-
- public void setParForParams(HashMap<String,String> params){
- _parforParams = params;
- }
-
- public static String[] createIterablePredicateVariables( String varName, Lop from, Lop to, Lop incr )
- {
- String[] ret = new String[4]; //varname, from, to, incr
-
- ret[0] = varName;
-
- if( from.getType()==Lop.Type.Data )
- ret[1] = from.getOutputParameters().getLabel();
- if( to.getType()==Lop.Type.Data )
- ret[2] = to.getOutputParameters().getLabel();
- if( incr.getType()==Lop.Type.Data )
- ret[3] = incr.getOutputParameters().getLabel();
-
- return ret;
- }
-
- private void checkNumericScalarOutput( Expression expr )
- throws LanguageException
- {
- if( expr == null || expr.getOutput() == null )
- return;
-
- Identifier ident = expr.getOutput();
- if( ident.getDataType() == DataType.MATRIX || ident.getDataType() == DataType.OBJECT ||
- (ident.getDataType() == DataType.SCALAR && (ident.getValueType() == ValueType.BOOLEAN ||
- ident.getValueType() == ValueType.STRING ||
- ident.getValueType() == ValueType.OBJECT)) )
- {
- LOG.error(this.printErrorLocation() + "expression in iterable predicate in for loop '" + expr.toString() + "' must return a numeric scalar");
- throw new LanguageException(this.printErrorLocation() + "expression in iterable predicate in for loop '" + expr.toString() + "' must return a numeric scalar");
- }
- }
-
-} // end class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/LanguageException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/LanguageException.java b/src/main/java/com/ibm/bi/dml/parser/LanguageException.java
deleted file mode 100644
index 6fd636a..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/LanguageException.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * <p>Exception occurring at the Language level.</p>
- */
-public class LanguageException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- /**
- * @see java.lang.Exception#Exception()
- */
- public LanguageException() {
- super();
- }
-
- /**
- * @see java.lang.Exception#Exception(String)
- */
- public LanguageException(String message) {
- super(message);
- }
-
- /**
- * @see java.lang.Exception#Exception(Throwable)
- */
- public LanguageException(Throwable cause) {
- super(cause);
- }
-
- /**
- * @see java.lang.Exception#Exception(String, Throwable)
- */
- public LanguageException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public LanguageException(String message, String code) {
- super(code + ERROR_MSG_DELIMITER + message);
- }
-
- public static class LanguageErrorCodes {
- public static final String UNSUPPORTED_EXPRESSION = "Unsupported Expression";
- public static final String INVALID_PARAMETERS = "Invalid Parameters";
- public static final String UNSUPPORTED_PARAMETERS = "Unsupported Parameters";
- public static final String GENERIC_ERROR = "Language Syntax Error";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/LiveVariableAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/LiveVariableAnalysis.java b/src/main/java/com/ibm/bi/dml/parser/LiveVariableAnalysis.java
deleted file mode 100644
index 735dc92..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/LiveVariableAnalysis.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-public abstract class LiveVariableAnalysis
-{
-
-
- VariableSet _read;
- VariableSet _updated;
- VariableSet _gen;
- VariableSet _kill;
- VariableSet _liveIn;
- VariableSet _liveOut;
- boolean _initialized = false;
-
- VariableSet _warnSet; // variables that may not be initialized
- // applicable for control blocks
-
- public VariableSet variablesRead(){
- return _read;
- }
-
- public VariableSet variablesUpdated() {
- return _updated;
- }
-
- public VariableSet getWarn(){
- return _warnSet;
- }
-
- public VariableSet liveIn() {
- return _liveIn;
- }
-
- public VariableSet liveOut() {
- return _liveOut;
- }
-
- public VariableSet getKill(){
- return _kill;
- }
-
- public void setLiveOut(VariableSet lo){
- _liveOut = lo;
- }
-
- public void setLiveIn(VariableSet li){
- _liveIn = li;
- }
-
- public void setUpdatedVariables( VariableSet vars ){
- _updated = vars;
- }
-
- public void setReadVariables( VariableSet vars ){
- _read = vars;
- }
-
- public abstract VariableSet initializeforwardLV(VariableSet activeIn) throws LanguageException;
- public abstract VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException;
- public abstract VariableSet analyze(VariableSet loPassed) throws LanguageException;
-
-
- public void updateLiveVariablesIn(VariableSet liveIn){
- updateLiveVariables(_liveIn,liveIn);
- }
-
- public void updateLiveVariablesOut(VariableSet liveOut){
- updateLiveVariables(_liveOut,liveOut);
- }
-
- private void updateLiveVariables(VariableSet origVars, VariableSet newVars){
- for (String var : newVars.getVariables().keySet()){
- if (origVars.containsVariable(var)){
- DataIdentifier varId = newVars.getVariable(var);
- if (varId != null){
- origVars.addVariable(var, varId);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/MultiAssignmentStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/MultiAssignmentStatement.java b/src/main/java/com/ibm/bi/dml/parser/MultiAssignmentStatement.java
deleted file mode 100644
index 66e48b1..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/MultiAssignmentStatement.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-
-
-
-public class MultiAssignmentStatement extends Statement
-{
-
- private ArrayList<DataIdentifier> _targetList;
- private Expression _source;
-
- // rewrites statement to support function inlining (creates deep copy)
- public Statement rewriteStatement(String prefix) throws LanguageException{
-
- ArrayList<DataIdentifier> newTargetList = new ArrayList<DataIdentifier>();
-
- // rewrite targetList (deep copy)
- for (DataIdentifier target : _targetList){
- DataIdentifier newTarget = (DataIdentifier) target.rewriteExpression(prefix);
- newTargetList.add(newTarget);
- }
-
- // rewrite source (deep copy)
- Expression newSource = _source.rewriteExpression(prefix);
-
- // create rewritten assignment statement (deep copy)
- MultiAssignmentStatement retVal = new MultiAssignmentStatement(newTargetList, newSource);
- retVal.setBeginLine(this.getBeginLine());
- retVal.setBeginColumn(this.getBeginColumn());
- retVal.setEndLine(this.getEndLine());
- retVal.setEndColumn(this.getEndColumn());
-
- return retVal;
- }
-
- public MultiAssignmentStatement(ArrayList<DataIdentifier> tList, Expression s){
- _targetList = tList;
- _source = s;
- }
-
- // NOTE: f is not used -- however, error is thrown "methods have same erasure" if not included in signature
- public MultiAssignmentStatement(ArrayList<ArrayList<Expression>> exprListList, Expression s, int f){
-
- _source = s;
-
- _targetList = new ArrayList<DataIdentifier>();
- for (ArrayList<Expression> exprList : exprListList){
- Expression expr = exprList.get(0);
- if( expr instanceof IndexedIdentifier )
- _targetList.add((IndexedIdentifier)expr);
- else
- _targetList.add(new DataIdentifier(expr.toString()));
- }
- }
-
- public ArrayList<DataIdentifier> getTargetList(){
- return _targetList;
- }
-
- public Expression getSource(){
- return _source;
- }
-
- @Override
- // conservative assignment to separate statement block; will merge later if possible
- public boolean controlStatement() {
- // ensure that breakpoints end up in own statement block
- if (DMLScript.ENABLE_DEBUG_MODE) {
- DMLBreakpointManager.insertBreakpoint(_source.getBeginLine());
- }
- return true;
- }
-
- public void initializeforwardLV(VariableSet activeIn){}
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- // add variables read by source expression
- result.addVariables(_source.variablesRead());
-
- // for any IndexedIdentifier on LHS, add variables for indexing expressions
- for (int i=0; i<_targetList.size(); i++){
- if (_targetList.get(i) instanceof IndexedIdentifier) {
- IndexedIdentifier target = (IndexedIdentifier) _targetList.get(i);
- result.addVariables(target.variablesRead());
- }
- }
-
- return result;
- }
-
- public VariableSet variablesUpdated() {
-
- VariableSet result = new VariableSet();
-
- // add target to updated list
- for (DataIdentifier target : _targetList){
- result.addVariable(target.getName(), target);
- }
- return result;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("[");
-
- for( int i=0; i< _targetList.size(); i++ )
- {
- sb.append(_targetList.get(i).toString());
- if (i < _targetList.size() - 1)
- sb.append(",");
- }
- sb.append("] = ");
- sb.append(_source.toString());
- sb.append(";");
-
- return sb.toString();
- }
-
- public void setSource(FunctionCallIdentifier s) {
- _source = s;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/OutputStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/OutputStatement.java b/src/main/java/com/ibm/bi/dml/parser/OutputStatement.java
deleted file mode 100644
index 3b9d6d6..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/OutputStatement.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-import com.ibm.bi.dml.parser.Expression.DataOp;
-
-
-public class OutputStatement extends Statement
-{
-
- private DataIdentifier _id;
- private DataExpression _paramsExpr;
-
- public static final String[] WRITE_VALID_PARAM_NAMES = { DataExpression.IO_FILENAME,
- DataExpression.FORMAT_TYPE,
- DataExpression.DELIM_DELIMITER,
- DataExpression.DELIM_HAS_HEADER_ROW,
- DataExpression.DELIM_SPARSE};
-
- public DataIdentifier getIdentifier(){
- return _id;
- }
-
- public DataExpression getSource(){
- return _paramsExpr;
- }
-
- public void setIdentifier(DataIdentifier t) {
- _id = t;
- }
-
- public OutputStatement(DataIdentifier t, DataOp op,
- String filename, int blp, int bcp, int elp, int ecp){
- _id = t;
- _paramsExpr = new DataExpression(op, new HashMap<String,Expression>(),
- filename, blp, bcp, elp, ecp);
- }
-
- /**
- * Called by the parser (both javacc and antlr).
- *
- * @param fname
- * @param fci
- * @param filename
- * @param blp
- * @param bcp
- * @param elp
- * @param ecp
- * @throws DMLParseException
- */
- OutputStatement(String fname, FunctionCallIdentifier fci,
- String filename, int blp, int bcp, int elp, int ecp)
- throws DMLParseException
- {
-
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- DataOp op = Expression.DataOp.WRITE;
- ArrayList<ParameterExpression> passedExprs = fci.getParamExprs();
- _paramsExpr = new DataExpression(op, new HashMap<String,Expression>(),
- filename, blp, bcp, elp, ecp);
- DMLParseException runningList = new DMLParseException(fname);
-
- //check number parameters and proceed only if this will not cause errors
- if (passedExprs.size() < 2)
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "write method must specify both variable to write to file, and filename to write variable to"));
- else
- {
- ParameterExpression firstParam = passedExprs.get(0);
- if (firstParam.getName() != null || (!(firstParam.getExpr() instanceof DataIdentifier)))
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "first argument to write method must be name of variable to be written out"));
- else
- _id = (DataIdentifier)firstParam.getExpr();
-
- ParameterExpression secondParam = passedExprs.get(1);
- if (secondParam.getName() != null || (secondParam.getName() != null && secondParam.getName().equals(DataExpression.IO_FILENAME)))
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "second argument to write method must be filename of file variable written to"));
- else
- addExprParam(DataExpression.IO_FILENAME, secondParam.getExpr(), false);
-
- for (int i = 2; i< passedExprs.size(); i++){
- ParameterExpression currParam = passedExprs.get(i);
- try {
- addExprParam(currParam.getName(), currParam.getExpr(), false);
- } catch (DMLParseException e){
- runningList.add(e);
- }
- }
- if (fname.equals("writeMM")){
- StringIdentifier writeMMExpr = new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- addExprParam(DataExpression.FORMAT_TYPE, writeMMExpr, false);
- }
- else if (fname.equals("write.csv")){
- StringIdentifier delimitedExpr = new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- addExprParam(DataExpression.FORMAT_TYPE, delimitedExpr, false);
- }
- }
-
- if (runningList.size() > 0)
- throw runningList;
- }
-
- public void setExprParam(String name, Expression value) {
- _paramsExpr.addVarParam(name, value);
- }
-
- public static boolean isValidParamName(String key){
- for (String paramName : WRITE_VALID_PARAM_NAMES)
- if (paramName.equals(key))
- return true;
- return false;
- }
-
- public void addExprParam(String name, Expression value, boolean fromMTDFile) throws DMLParseException
- {
- DMLParseException runningList = new DMLParseException(value.getFilename());
-
- if( _paramsExpr.getVarParam(name) != null )
- runningList.add(new DMLParseException(value.getFilename(), value.printErrorLocation() + "attempted to add IOStatement parameter " + name + " more than once"));
-
- if( !OutputStatement.isValidParamName(name) )
- runningList.add(new DMLParseException(value.getFilename(), value.printErrorLocation() + "attempted to add invalid write statement parameter: " + name));
-
- _paramsExpr.addVarParam(name, value);
-
- if (runningList.size() > 0)
- throw runningList;
- }
-
- // rewrites statement to support function inlining (create deep copy)
- public Statement rewriteStatement(String prefix) throws LanguageException{
-
- OutputStatement newStatement = new OutputStatement(null,Expression.DataOp.WRITE,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- // rewrite outputStatement variable name (creates deep copy)
- newStatement._id = (DataIdentifier)this._id.rewriteExpression(prefix);
-
- // rewrite parameter expressions (creates deep copy)
- DataOp op = _paramsExpr.getOpCode();
- HashMap<String,Expression> newExprParams = new HashMap<String,Expression>();
- for (String key : _paramsExpr.getVarParams().keySet()){
- Expression newExpr = _paramsExpr.getVarParam(key).rewriteExpression(prefix);
- newExprParams.put(key, newExpr);
- }
- DataExpression newParamerizedExpr = new DataExpression(op, newExprParams,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- newStatement.setExprParams(newParamerizedExpr);
- return newStatement;
- }
-
- public void setExprParams(DataExpression newParamerizedExpr) {
- _paramsExpr = newParamerizedExpr;
- }
- public Expression getExprParam(String key){
- return _paramsExpr.getVarParam(key);
- }
-
- public void initializeforwardLV(VariableSet activeIn){}
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(Statement.OUTPUTSTATEMENT + " ( " );
- sb.append( _id.toString() + ", " + _paramsExpr.getVarParam(DataExpression.IO_FILENAME).toString());
- for (String key : _paramsExpr.getVarParams().keySet()){
- if (!key.equals(DataExpression.IO_FILENAME))
- sb.append(", " + key + "=" + _paramsExpr.getVarParam(key));
- }
- sb.append(" );");
- return sb.toString();
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- // handle variable that is being written out
- result.addVariables(_id.variablesRead());
-
- // handle variables for output filename expression
- //result.addVariables(_filenameExpr.variablesRead());
-
- // add variables for parameter expressions
- for (String key : _paramsExpr.getVarParams().keySet())
- result.addVariables(_paramsExpr.getVarParam(key).variablesRead()) ;
-
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public boolean controlStatement() {
- // ensure that breakpoints end up in own statement block
- if (DMLScript.ENABLE_DEBUG_MODE) {
- DMLBreakpointManager.insertBreakpoint(_paramsExpr.getBeginLine());
- return true;
- }
-
- Expression fmt = _paramsExpr.getVarParam(DataExpression.FORMAT_TYPE);
- if ( fmt != null && fmt.toString().equalsIgnoreCase("csv")) {
- return true;
- }
- return false;
- }
-}
[14/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/Dml.g4 b/src/main/java/com/ibm/bi/dml/parser/antlr4/Dml.g4
deleted file mode 100644
index 4911088..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/Dml.g4
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-grammar Dml;
-
-@header
-{
- // Commenting the package name and explicitly passing it in build.xml to maintain compatibility with maven plugin
- // package com.ibm.bi.dml.antlr4;
-}
-
-// DML Program is a list of expression
-// For now, we only allow global function definitions (not nested or inside a while block)
-dmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* EOF;
-
-statement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // ImportStatement
- 'source' '(' filePath = STRING ')' 'as' namespace=ID ';'* # ImportStatement
- | 'setwd' '(' pathValue = STRING ')' ';'* # PathStatement
- // ------------------------------------------
- // Treat function call as AssignmentStatement or MultiAssignmentStatement
- // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
- // Convert FunctionCallIdentifier(paramExprs, ..) -> source
- | // TODO: Throw an informative error if user doesnot provide the optional assignment
- ( targetList+=dataIdentifier ('='|'<-') )? name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallAssignmentStatement
- | '[' targetList+=dataIdentifier (',' targetList+=dataIdentifier)* ']' ('='|'<-') name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallMultiAssignmentStatement
- // {notifyErrorListeners("Too many parentheses");}
- // ------------------------------------------
- // AssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') 'ifdef' '(' commandLineParam=dataIdentifier ',' source=expression ')' ';'* # IfdefAssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') source=expression ';'* # AssignmentStatement
- // ------------------------------------------
- // We don't support block statement
- // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
- // ------------------------------------------
- // IfStatement
- | 'if' '(' predicate=expression ')' (ifBody+=statement ';'* | '{' (ifBody+=statement ';'*)* '}') ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
- // ------------------------------------------
- // ForStatement & ParForStatement
- | 'for' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'* )* '}') # ForStatement
- // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
- | 'parfor' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # ParForStatement
- | 'while' '(' predicate=expression ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # WhileStatement
- // ------------------------------------------
-;
-
-iterablePredicate returns [ ExpressionInfo info ]
- @init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- } :
- from=expression ':' to=expression #IterablePredicateColonExpression
- | ID '(' from=expression ',' to=expression ',' increment=expression ')' #IterablePredicateSeqExpression
- ;
-
-functionStatement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // FunctionStatement & ExternalFunctionStatement
- // small change: only allow typed arguments here ... instead of data identifier
- name=ID ('<-'|'=') 'function' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? '{' (body+=statement ';'*)* '}' # InternalFunctionDefExpression
- | name=ID ('<-'|'=') 'externalFunction' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? 'implemented' 'in' '(' ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? ')' ';'* # ExternalFunctionDefExpression
- // ------------------------------------------
-;
-
-
-// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
-dataIdentifier returns [ ExpressionInfo dataInfo ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new com.ibm.bi.dml.parser.DataIdentifier();
-} :
- // ------------------------------------------
- // IndexedIdentifier
- name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? ']' # IndexedExpression
- // ------------------------------------------
- | ID # SimpleDataIdentifierExpression
- | COMMANDLINE_NAMED_ID # CommandlineParamExpression
- | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
-;
-expression returns [ ExpressionInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- // $info.expr = new com.ibm.bi.dml.parser.BinaryExpression(com.ibm.bi.dml.parser.Expression.BinaryOp.INVALID);
-} :
- // ------------------------------------------
- // BinaryExpression
- // power
- <assoc=right> left=expression op='^' right=expression # PowerExpression
- // unary plus and minus
- | op=('-'|'+') left=expression # UnaryExpression
- // sequence - since we are only using this into for
- //| left=expression op=':' right=expression # SequenceExpression
- // matrix multiply
- | left=expression op='%*%' right=expression # MatrixMulExpression
- // modulus and integer division
- | left=expression op=('%/%' | '%%' ) right=expression # ModIntDivExpression
- // arithmetic multiply and divide
- | left=expression op=('*'|'/') right=expression # MultDivExpression
- // arithmetic addition and subtraction
- | left=expression op=('+'|'-') right=expression # AddSubExpression
- // ------------------------------------------
- // RelationalExpression
- | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
- // ------------------------------------------
- // BooleanExpression
- // boolean not
- | op='!' left=expression # BooleanNotExpression
- // boolean and
- | left=expression op=('&'|'&&') right=expression # BooleanAndExpression
- // boolean or
- | left=expression op=('|'|'||') right=expression # BooleanOrExpression
-
- // ---------------------------------
- // only applicable for builtin function expressions
- | name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # BuiltinFunctionExpression
-
- // 4. Atomic
- | '(' left=expression ')' # AtomicExpression
-
- // Should you allow indexed expression here ?
- // | '[' targetList+=expression (',' targetList+=expression)* ']' # MultiIdExpression
-
- // | BOOLEAN # ConstBooleanIdExpression
- | 'TRUE' # ConstTrueExpression
- | 'FALSE' # ConstFalseExpression
- | INT # ConstIntIdExpression
- | DOUBLE # ConstDoubleIdExpression
- | STRING # ConstStringIdExpression
- | dataIdentifier # DataIdExpression
- // Special
- // | 'NULL' | 'NA' | 'Inf' | 'NaN'
-;
-
-typedArgNoAssign : paramType=ml_type paramName=ID;
-parameterizedExpression : (paramName=ID '=')? paramVal=expression;
-strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
-strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
-ID : (ALPHABET (ALPHABET|DIGIT|'_')* '::')? ALPHABET (ALPHABET|DIGIT|'_')*
- // Special ID cases:
- // | 'matrix' // --> This is a special case which causes lot of headache
- | 'as.scalar' | 'as.matrix' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail'
-;
-// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
-// Therefore, ugly work around for checking datatype
-ml_type : valueType | dataType '[' valueType ']';
-// Note to reduce number of keywords, these are case-sensitive,
-// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
-valueType: 'int' | 'integer' | 'string' | 'boolean' | 'double'
- | 'Int' | 'Integer' | 'String' | 'Boolean' | 'Double';
-dataType:
- // 'scalar' # ScalarDataTypeDummyCheck
- // |
- ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
- //| 'matrix' //---> See ID, this causes lot of headache
- ;
-INT : DIGIT+ [Ll]?;
-// BOOLEAN : 'TRUE' | 'FALSE';
-DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
-| DIGIT+ EXP? [Ll]?
-| '.' DIGIT+ EXP? [Ll]?
-;
-DIGIT: '0'..'9';
-ALPHABET : [a-zA-Z] ;
-fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
-COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
-COMMANDLINE_POSITION_ID: '$' DIGIT+;
-
-// supports single and double quoted string with escape characters
-STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
-fragment ESC : '\\' [abtnfrv"'\\] ;
-// Comments, whitespaces and new line
-LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
-MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
-WHITESPACE : (' ' | '\t' | '\r' | '\n')+ -> skip ;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticErrorListener.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticErrorListener.java
deleted file mode 100644
index 5fdf2d8..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticErrorListener.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Recognizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-
-import java.util.Stack;
-
-public class DmlSyntacticErrorListener {
-
-
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static class CustomDmlErrorListener extends BaseErrorListener {
-
- private boolean atleastOneError = false;
- private Stack<String> currentFileName = new Stack<String>();
-
- public void pushCurrentFileName(String currentFilePath) {
- currentFileName.push(currentFilePath);
- }
-
- public String peekFileName() {
- return currentFileName.peek();
- }
-
- public String popFileName() {
- return currentFileName.pop();
- }
-
- public void validationError(int line, int charPositionInLine, String msg) {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty()) {
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- }
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public void validationWarning(int line, int charPositionInLine, String msg) {
- try {
- //atleastOneError = true; ---> not an error, just warning
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.warn("ERROR: while customizing error message:" + e1);
- }
- }
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
- int line, int charPositionInLine,
- String msg, RecognitionException e)
- {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public boolean isAtleastOneError() {
- return atleastOneError;
- }
-
- public void setAtleastOneError(boolean atleastOneError) {
- this.atleastOneError = atleastOneError;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidator.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidator.java
deleted file mode 100644
index f19f618..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidator.java
+++ /dev/null
@@ -1,1517 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.ErrorNode;
-import org.antlr.v4.runtime.tree.TerminalNode;
-
-import com.ibm.bi.dml.parser.ConditionalPredicate;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DoubleIdentifier;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.Expression.DataOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AddSubExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.AtomicExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanAndExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanNotExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BooleanOrExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.BuiltinFunctionExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.CommandlineParamExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.CommandlinePositionExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstDoubleIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstFalseExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstIntIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstStringIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ConstTrueExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DataIdExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DataIdentifierContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.DmlprogramContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ExternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ForStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallMultiAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IfStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IfdefAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ImportStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IndexedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.InternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IterablePredicateColonExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.IterablePredicateSeqExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MatrixDataTypeCheckContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MatrixMulExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.Ml_typeContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ModIntDivExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.MultDivExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParForStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.PathStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.PowerExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.RelationalExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.SimpleDataIdentifierExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StrictParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.StrictParameterizedKeyValueStringContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.TypedArgNoAssignContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.UnaryExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ValueTypeContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.WhileStatementContext;
-import com.ibm.bi.dml.parser.AssignmentStatement;
-import com.ibm.bi.dml.parser.BinaryExpression;
-import com.ibm.bi.dml.parser.BooleanExpression;
-import com.ibm.bi.dml.parser.BooleanIdentifier;
-import com.ibm.bi.dml.parser.BuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ConstIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.FunctionCallIdentifier;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.IndexedIdentifier;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.IterablePredicate;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.MultiAssignmentStatement;
-import com.ibm.bi.dml.parser.OutputStatement;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParameterExpression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.PathStatement;
-import com.ibm.bi.dml.parser.PrintStatement;
-import com.ibm.bi.dml.parser.RelationalExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.WhileStatement;
-
-public class DmlSyntacticValidator implements DmlListener
-{
- private DmlSyntacticValidatorHelper helper = null;
-
- private String _workingDir = "."; //current working directory
- private String _currentPath = null; //current file path
- private HashMap<String,String> argVals = null;
-
- public DmlSyntacticValidator(DmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
- this.helper = helper;
- this.argVals = argVals;
-
- _currentPath = currentPath;
- }
-
- // Functions we have to implement but don't really need it
- @Override
- public void enterAddSubExpression(AddSubExpressionContext ctx) { }
- @Override
- public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
- @Override
- public void enterAtomicExpression(AtomicExpressionContext ctx) { }
- @Override
- public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
- @Override
- public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
- @Override
- public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
- @Override
- public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
- @Override
- public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
- @Override
- public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
- @Override
- public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
- @Override
- public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
- @Override
- public void enterDataIdExpression(DataIdExpressionContext ctx) { }
-
- @Override
- public void enterDmlprogram(DmlprogramContext ctx) { }
- @Override
- public void enterEveryRule(ParserRuleContext arg0) {
- if(arg0 instanceof StatementContext) {
- if(((StatementContext) arg0).info == null) {
- ((StatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof FunctionStatementContext) {
- if(((FunctionStatementContext) arg0).info == null) {
- ((FunctionStatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof ExpressionContext) {
- if(((ExpressionContext) arg0).info == null) {
- ((ExpressionContext) arg0).info = new ExpressionInfo();
- }
- }
- if(arg0 instanceof DataIdentifierContext) {
- if(((DataIdentifierContext) arg0).dataInfo == null) {
- ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
- }
- }
- }
- @Override
- public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterForStatement(ForStatementContext ctx) {}
- @Override
- public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
- @Override
- public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
- @Override
- public void enterIfStatement(IfStatementContext ctx) { }
- @Override
- public void enterImportStatement(ImportStatementContext ctx) { }
- @Override
- public void enterIndexedExpression(IndexedExpressionContext ctx) { }
- @Override
- public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
- public void enterMatrixMulExpression(MatrixMulExpressionContext ctx) { }
- @Override
- public void enterMl_type(Ml_typeContext ctx) { }
- @Override
- public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
- @Override
- public void enterMultDivExpression(MultDivExpressionContext ctx) { }
- @Override
- public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
- @Override
- public void enterParForStatement(ParForStatementContext ctx) { }
- @Override
- public void enterPathStatement(PathStatementContext ctx) { }
- @Override
- public void enterPowerExpression(PowerExpressionContext ctx) { }
- @Override
- public void enterRelationalExpression(RelationalExpressionContext ctx) { }
- @Override
- public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
- @Override
- public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
- @Override
- public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterUnaryExpression(UnaryExpressionContext ctx) { }
- @Override
- public void enterValueType(ValueTypeContext ctx) { }
- @Override
- public void enterWhileStatement(WhileStatementContext ctx) { }
-
- @Override
- public void visitErrorNode(ErrorNode arg0) { }
- @Override
- public void visitTerminal(TerminalNode arg0) { }
- @Override
- public void exitEveryRule(ParserRuleContext arg0) {}
- // --------------------------------------------------------------------
- private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
- // expr.setFilename(helper.getCurrentFileName());
- String txt = ctx.getText();
- expr.setFilename(_currentPath);
- expr.setBeginLine(ctx.start.getLine());
- expr.setBeginColumn(ctx.start.getCharPositionInLine());
- expr.setEndLine(ctx.stop.getLine());
- expr.setEndColumn(ctx.stop.getCharPositionInLine());
- if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
- expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
- }
- }
-
- private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
- String txt = ctx.getText();
- stmt.setFilename(helper.getCurrentFileName());
- stmt.setBeginLine(ctx.start.getLine());
- stmt.setBeginColumn(ctx.start.getCharPositionInLine());
- stmt.setEndLine(ctx.stop.getLine());
- stmt.setEndColumn(ctx.stop.getCharPositionInLine());
- if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
- stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
- }
- }
-
- // For now do no type checking, let validation handle it.
- // This way parser doesn't have to open metadata file
- @Override
- public void exitAddSubExpression(AddSubExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
- @Override
- public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitUnaryExpression(UnaryExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- if(ctx.left.info.expr instanceof IntIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else if(ctx.left.info.expr instanceof DoubleIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else {
- Expression right = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.op.getText().compareTo("-") == 0) {
- right = new IntIdentifier(-1, fileName, line, col, line, col);
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp("*");
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(right);
- }
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitMultDivExpression(MultDivExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- @Override
- public void exitPowerExpression(PowerExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitRelationalExpression(RelationalExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
-// String fileName = helper.getCurrentFileName();
-// int line = ctx.start.getLine();
-// int col = ctx.start.getCharPositionInLine();
-// ArrayList<ParameterExpression> paramExpression = new ArrayList<ParameterExpression>();
-// paramExpression.add(new ParameterExpression(null, ctx.left.info.expr));
-// paramExpression.add(new ParameterExpression(null, ctx.right.info.expr));
-// ParameterExpression operator = new ParameterExpression(null, new StringIdentifier(ctx.op.getText(), fileName, line, col, line, col));
-// paramExpression.add(operator);
-//
-// try {
-// BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression("ppred", paramExpression, fileName, line, col, line, col);
-// if (bife != null){
-// // It is a builtin function
-// ctx.info.expr = bife;
-// return;
-// }
-// }
-// catch(Exception e) {}
-// helper.notifyErrorListeners("Cannot parse relational expression", ctx.getStart());
-
- Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
- ctx.info.expr = new RelationalExpression(rop);
- ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitAtomicExpression(AtomicExpressionContext ctx) {
- ctx.info.expr = ctx.left.info.expr;
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-// @Override
-// public void exitConstBooleanIdExpression(ConstBooleanIdExpressionContext ctx) {
-// boolean val = false;
-// if(ctx.getText().compareTo("TRUE") == 0) {
-// val = true;
-// }
-// else if(ctx.getText().compareTo("FALSE") == 0) {
-// val = false;
-// }
-// else {
-// helper.notifyErrorListeners("cannot parse the boolean value: \'" + ctx.getText() + "\'", ctx.getStart());
-// return;
-// }
-// int linePosition = ctx.start.getLine();
-// int charPosition = ctx.start.getCharPositionInLine();
-// ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
- @Override
- public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
- try {
- double val = Double.parseDouble(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the double value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
- try {
- long val = Long.parseLong(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the integer value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
- String val = "";
- String text = ctx.getText();
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
- return;
- }
-
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitDataIdExpression(DataIdExpressionContext ctx) {
- ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
- setFileLineColumn(ctx.info.expr, ctx);
-// if(ctx.getChild(0) instanceof DataIdentifierContext) {
-// ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
-// }
-// else {
-// String msg = "cannot evaluate data expression ... strange";
-// helper.notifyErrorListeners(msg, ctx.start);
-// }
- }
-
- @Override
- public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
- // This is either a function, or variable with namespace
- // By default, it assigns to a data type
- ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- }
-
- @Override
- public void exitIndexedExpression(IndexedExpressionContext ctx) {
- ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- try {
- ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
-
- ArrayList<Expression> rowIndices = new ArrayList<Expression>();
- ArrayList<Expression> colIndices = new ArrayList<Expression>();
-
- boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
- boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
- boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
- boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
-
- if(!isRowLower && !isRowUpper) {
- // both not set
- rowIndices.add(null); rowIndices.add(null);
- }
- else if(isRowLower && isRowUpper) {
- // both set
- rowIndices.add(ctx.rowLower.info.expr);
- rowIndices.add(ctx.rowUpper.info.expr);
- }
- else if(isRowLower && !isRowUpper) {
- // only row set
- rowIndices.add(ctx.rowLower.info.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
- return;
- }
-
- if(!isColLower && !isColUpper) {
- // both not set
- colIndices.add(null); colIndices.add(null);
- }
- else if(isColLower && isColUpper) {
- colIndices.add(ctx.colLower.info.expr);
- colIndices.add(ctx.colUpper.info.expr);
- }
- else if(isColLower && !isColUpper) {
- colIndices.add(ctx.colLower.info.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
- return;
- }
-
-
-// boolean rowIndexLowerSet = false;
-// boolean colIndexLowerSet = false;
-//
-// if(ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null)) {
-// rowIndices.add(ctx.rowLower.info.expr);
-// rowIndexLowerSet = true;
-// }
-// else {
-// rowIndices.add(null);
-// }
-// if(ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null)) {
-// rowIndices.add(ctx.rowUpper.info.expr);
-// if(!rowIndexLowerSet) {
-// helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
-// return;
-// }
-// }
-// if(ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null)) {
-// colIndices.add(ctx.colLower.info.expr);
-// colIndexLowerSet = true;
-// }
-// else {
-// colIndices.add(null);
-// }
-// if(ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null)) {
-// colIndices.add(ctx.colUpper.info.expr);
-// if(!colIndexLowerSet) {
-// helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
-// return;
-// }
-// }
- exprList.add(rowIndices);
- exprList.add(colIndices);
- ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot set the indices", ctx.start);
- return;
- }
- }
-
- private ConstIdentifier getConstIdFromString(String varValue, Token start) {
- // Both varName and varValue are correct
- int linePosition = start.getLine();
- int charPosition = start.getCharPositionInLine();
- try {
- long val = Long.parseLong(varValue);
- return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e) {
- try {
- double val = Double.parseDouble(varValue);
- return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e1) {
- try {
- if(varValue.compareTo("TRUE") == 0 || varValue.compareTo("FALSE") == 0) {
- boolean val = false;
- if(varValue.compareTo("TRUE") == 0) {
- val = true;
- }
- return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- else {
- String val = "";
- String text = varValue;
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- val = text;
- // the commandline parameters can be passed without any quotes
-// helper.notifyErrorListeners("something wrong while parsing string ... strange", start);
-// return null;
- }
- return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- }
- catch(Exception e3) {
- helper.notifyErrorListeners("unable to cast the commandline parameter into int/double/boolean/string", start);
- return null;
- }
- }
- }
-
- }
- private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
-
- if(!varName.startsWith("$")) {
- helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
- return;
- }
-
- String varValue = null;
- for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
- if(arg.getKey().trim().compareTo(varName) == 0) {
- if(varValue != null) {
- helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
- return;
- }
- else {
- varValue = arg.getValue().trim();
- }
- }
- }
-
- if(varValue == null) {
- // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
- return;
- }
-
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- if(varValue.trim().compareTo("") == 0)
- return;
-
- dataInfo.expr = getConstIdFromString(varValue, start);
- }
-
- @Override
- public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
- }
-
- @Override
- public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
- }
-
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitImportStatement(ImportStatementContext ctx)
- {
- //prepare import filepath
- String filePath = ctx.filePath.getText();
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
- namespace = ctx.namespace.getText();
- }
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- //concatenate working directory to filepath
- filePath = _workingDir + File.separator + filePath;
-
- DMLProgram prog = null;
- try {
- prog = (new DMLParserWrapper()).doParse(filePath, null, argVals);
- } catch (ParseException e) {
- helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
- return;
- }
- // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
- if(prog == null) {
- helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
- return;
- }
- else {
- ctx.info.namespaces = new HashMap<String, DMLProgram>();
- ctx.info.namespaces.put(namespace, prog);
- ctx.info.stmt = new ImportStatement();
- ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
- ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
- ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
- }
- }
-
- @Override
- public void exitAssignmentStatement(AssignmentStatementContext ctx) {
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = ctx.source.info.expr;
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
- return;
- }
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
-
- private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
- try {
- ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid function call", ctx.start);
- return;
- }
- }
-
- private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
- return;
- }
- Expression expr = paramExpression.get(0).getExpr();
- if(expr == null) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- try {
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- }
-
- private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() < 2){
- helper.notifyErrorListeners("incorrect usage of write function (atleast 2 arguments required)", ctx.start);
- return;
- }
- if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
- // && paramExpression.get(0).getName() == null
- // correct usage of identifier
- // if(paramExpression.get(1).getName() == null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- HashMap<String, Expression> varParams = new HashMap<String, Expression>();
- varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
- for(int i = 2; i < paramExpression.size(); i++) {
- // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
- varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
- }
-
- DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
- ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
- return;
- //}
- }
-
- helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
- return;
-
- }
-
- @Override
- public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
- String fullyQualifiedFunctionName = ctx.name.getText();
- String [] fnNames = fullyQualifiedFunctionName.split("::");
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else {
- helper.notifyErrorListeners("incorrect function name", ctx.name);
- return;
- }
-
- if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setPrintStatement(ctx, functionName);
- return;
- }
- else if(functionName.compareTo("write") == 0
- && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setOutputStatement(ctx);
- return;
- }
-
- boolean ignoreLValue = false;
- if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
- helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
- return;
- }
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(!ignoreLValue) {
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
- if(!helper.validateBuiltinFunctions(ctx)) {
- return; // it is a built-in function and validation failed, so donot proceed ahead.
- }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setAssignmentStatement(target, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setAssignmentStatement(target, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setAssignmentStatement(target, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- }
-
- setAssignmentStatement(target, functCall, ctx);
- }
-
- @Override
- public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- String functionName = ctx.name.getText();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- ctx.info.expr = bife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- ctx.info.expr = pbife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (dbife != null){
- ctx.info.expr = dbife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
- }
-
- private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
- ctx.info.stmt = new MultiAssignmentStatement(target, expression);
- ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitFunctionCallMultiAssignmentStatement(
- FunctionCallMultiAssignmentStatementContext ctx) {
- String fullyQualifiedFunctionName = ctx.name.getText();
- String [] fnNames = fullyQualifiedFunctionName.split("::");
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else {
- helper.notifyErrorListeners("incorrect function name", ctx.name);
- return;
- }
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
- for(DataIdentifierContext dataCtx : ctx.targetList) {
- if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
- targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setMultiAssignmentStatement(targetList, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setMultiAssignmentStatement(targetList, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setMultiAssignmentStatement(targetList, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return;
- }
- }
-
- setMultiAssignmentStatement(targetList, functCall, ctx);
- }
-
- private StatementBlock getStatementBlock(Statement current) {
- return DMLParserWrapper.getStatementBlock(current);
- }
-
- @Override
- public void exitIfStatement(IfStatementContext ctx) {
- IfStatement ifStmt = new IfStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- ifStmt.setConditionalPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ifStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.ifBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.ifBody) {
- ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksIfBody();
- }
-
- if(ctx.elseBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.elseBody) {
- ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksElseBody();
- }
-
- ctx.info.stmt = ifStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitWhileStatement(WhileStatementContext ctx) {
- WhileStatement whileStmt = new WhileStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- whileStmt.setPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- whileStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- whileStmt.mergeStatementBlocks();
- }
-
- ctx.info.stmt = whileStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitForStatement(ForStatementContext ctx) {
- ForStatement forStmt = new ForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = null;
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.iterPred.info.increment != null) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- forStmt.setPredicate(predicate);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- forStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = forStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitParForStatement(ParForStatementContext ctx) {
- ParForStatement parForStmt = new ParForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = new HashMap<String, String>();
- if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
- for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
- parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
- }
- }
-
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
-
- if( ctx.iterPred.info.increment != null ) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- parForStmt.setPredicate(predicate);
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- parForStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = parForStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
-
- @Override
- public void exitDmlprogram(DmlprogramContext ctx) { }
-
-
- // ----------------------------------------------------------------------
- @Override
- public void exitValueType(ValueTypeContext ctx) { }
- @Override
- public void exitMl_type(Ml_typeContext ctx) { }
-
- private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
- ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
- for(TypedArgNoAssignContext paramCtx : ctx) {
- DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
- String dataType = null;
- String valueType = null;
-
- if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
- || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
- dataType = "scalar";
- }
- else {
- dataType = paramCtx.paramType.dataType().getText();
- }
-
- if(dataType.compareTo("matrix") == 0 || dataType.compareTo("Matrix") == 0) {
- // matrix
- dataId.setDataType(DataType.MATRIX);
- }
- else if(dataType.compareTo("scalar") == 0 || dataType.compareTo("Scalar") == 0) {
- // scalar
- dataId.setDataType(DataType.SCALAR);
- }
- else {
- helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
- return null;
- }
-
- valueType = paramCtx.paramType.valueType().getText();
- if(valueType.compareTo("int") == 0 || valueType.compareTo("integer") == 0
- || valueType.compareTo("Int") == 0 || valueType.compareTo("Integer") == 0) {
- dataId.setValueType(ValueType.INT);
- }
- else if(valueType.compareTo("string") == 0 || valueType.compareTo("String") == 0) {
- dataId.setValueType(ValueType.STRING);
- }
- else if(valueType.compareTo("boolean") == 0 || valueType.compareTo("Boolean") == 0) {
- dataId.setValueType(ValueType.BOOLEAN);
- }
- else if(valueType.compareTo("double") == 0 || valueType.compareTo("Double") == 0) {
- dataId.setValueType(ValueType.DOUBLE);
- }
- else if(valueType.compareTo("bool") == 0) {
- helper.notifyErrorListeners("invalid valuetype " + valueType + " (Quickfix: use \'boolean\' instead)", paramCtx.start);
- return null;
- }
- else {
- helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
- return null;
- }
- retVal.add(dataId);
- }
- return retVal;
- }
-
- @Override
- public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
- FunctionStatement functionStmt = new FunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
-
- if(ctx.body.size() > 0) {
- // handle function body
- // Create arraylist of one statement block
- ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
- for(StatementContext stmtCtx : ctx.body) {
- body.add(getStatementBlock(stmtCtx.info.stmt));
- }
- functionStmt.setBody(body);
- functionStmt.mergeStatementBlocks();
- }
- else {
- helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
- return;
- }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
- @Override
- public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
- ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
- // set other parameters
- HashMap<String, String> otherParams = new HashMap<String,String>();
- boolean atleastOneClassName = false;
- for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
- String paramName = otherParamCtx.paramName.getText();
- String val = "";
- String text = otherParamCtx.paramVal.getText();
- // First unquote the string
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- // Empty value allowed
- }
- else {
- helper.notifyErrorListeners("the value of user parameter for external function should be of type string", ctx.start);
- return;
- }
- otherParams.put(paramName, val);
- if(paramName.compareTo("classname") == 0) {
- atleastOneClassName = true;
- }
- }
- functionStmt.setOtherParams(otherParams);
- if(!atleastOneClassName) {
- helper.notifyErrorListeners("the parameter \'className\' needs to be passed for externalFunction", ctx.start);
- return;
- }
-
-// if(ctx.body.size() > 0) {
-// // handle function body
-// // Create arraylist of one statement block
-// ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
-// for(StatementContext stmtCtx : ctx.body) {
-// body.add(getStatementBlock(stmtCtx.info.stmt));
-// }
-// ((ExternalFunctionStatement) functionStmt).setBody(body);
-// ((ExternalFunctionStatement) functionStmt).mergeStatementBlocks();
-// }
-// else {
-// helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
-// return;
-// }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
-
- @Override
- public void exitPathStatement(PathStatementContext ctx) {
- PathStatement stmt = new PathStatement(ctx.pathValue.getText());
- String filePath = ctx.pathValue.getText();
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- _workingDir = filePath;
- ctx.info.stmt = stmt;
- }
-
- @Override
- public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
- if(!ctx.commandLineParam.getText().startsWith("$")) {
- helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
- return;
- }
-
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = null;
- if(ctx.commandLineParam.dataInfo.expr != null) {
- // Since commandline parameter is set
- // The check of following is done in fillExpressionInfoCommandLineParameters:
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- source = ctx.commandLineParam.dataInfo.expr;
- }
- else {
- source = ctx.source.info.expr;
- }
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
- return;
- }
-
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
- // ----------------------------------------------------------------------
- @Override
- public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
-
-
- @Override
- public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
-
- @Override
- public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
- @Override
- public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
- @Override
- public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("matrix") == 0
- || ctx.ID().getText().compareTo("Matrix") == 0
- || ctx.ID().getText().compareTo("Scalar") == 0
- || ctx.ID().getText().compareTo("scalar") == 0
- ) {
- // Do nothing
- }
- else {
- helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
- }
- }
-
- @Override
- public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
- @Override
- public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
- @Override
- public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
- @Override
- public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
- @Override
- public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
-
- @Override
- public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = null;
- }
-
- @Override
- public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
- if(ctx.ID().getText().compareTo("seq") != 0) {
- helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'seq\'", ctx.start);
- return;
- }
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = ctx.increment.info.expr;
- }
-
- @Override
- public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
- @Override
- public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
-
- @Override
- public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
- boolean val = false;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
- @Override
- public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
- boolean val = true;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidatorHelper.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidatorHelper.java
deleted file mode 100644
index 1d0f675..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/DmlSyntacticValidatorHelper.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.v4.runtime.Token;
-
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-
-public class DmlSyntacticValidatorHelper {
-
- private CustomDmlErrorListener _errorListener = null;
-
- public DmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
- this._errorListener = errorListener;
- }
-
- public void notifyErrorListeners(String message, int line, int charPositionInLine) {
- this._errorListener.validationError(line, charPositionInLine, message);
- }
-
- public void notifyErrorListeners(String message, Token op) {
- this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public String getCurrentFileName() {
- return _errorListener.peekFileName();
- }
-
-// public static void setInfoForArithmeticOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// try {
-// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// com.ibm.bi.dml.parser.Expression.BinaryOp bop = com.ibm.bi.dml.parser.Expression.getBinaryOp(opStr);
-// current = new com.ibm.bi.dml.parser.BinaryExpression(bop);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-// catch(Exception e) {
-// System.out.println("In setInfoForArithmeticOp>>");
-// e.printStackTrace();
-// }
-// }
-
-// public static void setInfoForBooleanOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// com.ibm.bi.dml.parser.Expression.BooleanOp bop = com.ibm.bi.dml.parser.Expression.getBooleanOp(opStr);
-// current = new com.ibm.bi.dml.parser.BooleanExpression(bop);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-
- public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
- String functionName = ctx.name.getText().replaceAll(" ", "").trim();
- if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + "::write") == 0) {
- return validateBuiltinWriteFunction(ctx);
- }
- return true;
- }
-
- private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
-
- return true;
- }
-
- public ArrayList<com.ibm.bi.dml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<com.ibm.bi.dml.parser.ParameterExpression> retVal = new ArrayList<com.ibm.bi.dml.parser.ParameterExpression>();
- for(ParameterizedExpressionContext ctx : paramExprs) {
- String paramName = null;
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- paramName = ctx.paramName.getText();
- }
- com.ibm.bi.dml.parser.ParameterExpression myArg = new com.ibm.bi.dml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
- retVal.add(myArg);
- }
- return retVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/ExpressionInfo.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/ExpressionInfo.java
deleted file mode 100644
index d7ea7eb..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/ExpressionInfo.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-
-public class ExpressionInfo {
-
- public com.ibm.bi.dml.parser.Expression expr = null;
-
- // For parfor and for
- public com.ibm.bi.dml.parser.Expression from = null;
- public com.ibm.bi.dml.parser.Expression to = null;
- public com.ibm.bi.dml.parser.Expression increment = null;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/antlr4/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/antlr4/StatementInfo.java b/src/main/java/com/ibm/bi/dml/parser/antlr4/StatementInfo.java
deleted file mode 100644
index e32e21a..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/antlr4/StatementInfo.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.antlr4;
-
-import java.util.HashMap;
-import com.ibm.bi.dml.parser.DMLProgram;
-
-public class StatementInfo {
-
- public com.ibm.bi.dml.parser.Statement stmt = null;
-
- // Valid only for import statements
- public HashMap<String,DMLProgram> namespaces = null;
-
- // Valid only for function statement
- //public String namespace = DMLProgram.DEFAULT_NAMESPACE;
- public String functionName = "";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/ExpressionInfo.java b/src/main/java/com/ibm/bi/dml/parser/python/ExpressionInfo.java
deleted file mode 100644
index 09e8c27..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/ExpressionInfo.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.ibm.bi.dml.parser.python;
-
-/**
- * This class exists solely to prevent compiler warnings.
- *
- * <p>
- * The ExpressionInfo and StatementInfo classes are shared among both parsers
- * (R-like and Python-like dialects), and Antlr-generated code assumes that
- * these classes are present in the parser's namespace.
- */
-class ExpressionInfo extends com.ibm.bi.dml.parser.antlr4.ExpressionInfo {
-
-}
[37/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimator.java b/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimator.java
deleted file mode 100644
index 1f6d947..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimator.java
+++ /dev/null
@@ -1,787 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.cost;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.StringTokenizer;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateTernaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateUnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DataGenCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MMTSJCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.StringInitCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.UnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-public abstract class CostEstimator
-{
-
- protected static final Log LOG = LogFactory.getLog(CostEstimator.class.getName());
-
- private static final int DEFAULT_NUMITER = 15;
-
- protected static final VarStats _unknownStats = new VarStats(1,1,-1,-1,-1,false);
- protected static final VarStats _scalarStats = new VarStats(1,1,1,1,1,true);
-
- /**
- *
- * @param rtprog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public double getTimeEstimate(Program rtprog, LocalVariableMap vars, HashMap<String,VarStats> stats)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- double costs = 0;
-
- //obtain stats from symboltable (e.g., during recompile)
- maintainVariableStatistics(vars, stats);
-
- //get cost estimate
- for( ProgramBlock pb : rtprog.getProgramBlocks() )
- costs += rGetTimeEstimate(pb, stats, new HashSet<String>(), true);
-
- return costs;
- }
-
- /**
- *
- * @param pb
- * @param vars
- * @param stats
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public double getTimeEstimate(ProgramBlock pb, LocalVariableMap vars, HashMap<String,VarStats> stats, boolean recursive)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //obtain stats from symboltable (e.g., during recompile)
- maintainVariableStatistics(vars, stats);
-
- //get cost estimate
- return rGetTimeEstimate(pb, stats, new HashSet<String>(), recursive);
- }
-
-
- /**
- *
- * @param hops
- * @param vars
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws IOException
- * @throws LopsException
- * @throws HopsException
- */
- public double getTimeEstimate( ArrayList<Hop> hops, LocalVariableMap vars, HashMap<String,VarStats> stats )
- throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException, LopsException, IOException
- {
- double costs = 0;
-
- ArrayList<Instruction> linst = Recompiler.recompileHopsDag(null, hops, vars, null, false, 0);
- ProgramBlock pb = new ProgramBlock(null);
- pb.setInstructions(linst);
-
- //obtain stats from symboltable (e.g., during recompile)
- maintainVariableStatistics(vars, stats);
-
- //get cost estimate
- costs = rGetTimeEstimate(pb, stats, new HashSet<String>(), true);
-
- return costs;
- }
-
- /**
- *
- * @param pb
- * @param vars
- * @param stats
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private double rGetTimeEstimate(ProgramBlock pb, HashMap<String,VarStats> stats, HashSet<String> memoFunc, boolean recursive)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- double ret = 0;
-
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock tmp = (WhileProgramBlock)pb;
- if( recursive )
- for (ProgramBlock pb2 : tmp.getChildBlocks())
- ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
- ret *= DEFAULT_NUMITER;
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock tmp = (IfProgramBlock)pb;
- if( recursive ) {
- for( ProgramBlock pb2 : tmp.getChildBlocksIfBody() )
- ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
- if( tmp.getChildBlocksElseBody()!=null )
- for( ProgramBlock pb2 : tmp.getChildBlocksElseBody() ){
- ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
- ret /= 2; //weighted sum
- }
- }
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock tmp = (ForProgramBlock)pb;
- if( recursive )
- for( ProgramBlock pb2 : tmp.getChildBlocks() )
- ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
-
- ret *= getNumIterations(stats, tmp.getIterablePredicateVars());
- }
- else if ( pb instanceof FunctionProgramBlock
- && !(pb instanceof ExternalFunctionProgramBlock)) //see generic
- {
- FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
- if( recursive )
- for( ProgramBlock pb2 : tmp.getChildBlocks() )
- ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
- }
- else
- {
- ArrayList<Instruction> tmp = pb.getInstructions();
-
- for( Instruction inst : tmp )
- {
- if( inst instanceof CPInstruction ) //CP
- {
- //obtain stats from createvar, cpvar, rmvar, rand
- maintainCPInstVariableStatistics((CPInstruction)inst, stats);
-
- //extract statistics (instruction-specific)
- Object[] o = extractCPInstStatistics(inst, stats);
- VarStats[] vs = (VarStats[]) o[0];
- String[] attr = (String[]) o[1];
-
- //if(LOG.isDebugEnabled())
- // LOG.debug(inst);
-
- //call time estimation for inst
- ret += getCPInstTimeEstimate(inst, vs, attr);
-
- if( inst instanceof FunctionCallCPInstruction ) //functions
- {
- FunctionCallCPInstruction finst = (FunctionCallCPInstruction)inst;
- String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName());
- //awareness of recursive functions, missing program
- if( !memoFunc.contains(fkey) && pb.getProgram()!=null )
- {
- if(LOG.isDebugEnabled())
- LOG.debug("Begin Function "+fkey);
-
- memoFunc.add(fkey);
- Program prog = pb.getProgram();
- FunctionProgramBlock fpb = prog.getFunctionProgramBlock(
- finst.getNamespace(), finst.getFunctionName());
- ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive);
- memoFunc.remove(fkey);
-
- if(LOG.isDebugEnabled())
- LOG.debug("End Function "+fkey);
- }
- }
- }
- else if(inst instanceof MRJobInstruction) //MR
- {
- //obtain stats for job
- maintainMRJobInstVariableStatistics(inst, stats);
-
- //extract input statistics
- Object[] o = extractMRJobInstStatistics(inst, stats);
- VarStats[] vs = (VarStats[]) o[0];
-
- //if(LOG.isDebugEnabled())
- // LOG.debug(inst);
-
- if(LOG.isDebugEnabled())
- LOG.debug("Begin MRJob type="+((MRJobInstruction)inst).getJobType());
-
- //call time estimation for complex MR inst
- ret += getMRJobInstTimeEstimate(inst, vs, null);
-
- if(LOG.isDebugEnabled())
- LOG.debug("End MRJob");
-
- //cleanup stats for job
- cleanupMRJobVariableStatistics(inst, stats);
- }
- }
- }
-
- return ret;
- }
-
-
- /**
- *
- * @param vars
- * @param stats
- * @throws DMLRuntimeException
- */
- private void maintainVariableStatistics( LocalVariableMap vars, HashMap<String, VarStats> stats )
- throws DMLRuntimeException
- {
- for( String varname : vars.keySet() )
- {
- Data dat = vars.get(varname);
- VarStats vs = null;
- if( dat instanceof MatrixObject ) //matrix
- {
- MatrixObject mo = (MatrixObject) dat;
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)mo.getMetaData()).getMatrixCharacteristics();
- long rlen = mc.getRows();
- long clen = mc.getCols();
- long brlen = mc.getRowsPerBlock();
- long bclen = mc.getColsPerBlock();
- long nnz = mc.getNonZeros();
- boolean inmem = mo.getStatusAsString().equals("CACHED");
- vs = new VarStats(rlen, clen, brlen, bclen, nnz, inmem);
- }
- else //scalar
- {
- vs = _scalarStats;
- }
-
- stats.put(varname, vs);
- //System.out.println(varname+" "+vs);
- }
- }
-
- /**
- *
- * @param inst
- * @param stats
- */
- private void maintainCPInstVariableStatistics( CPInstruction inst, HashMap<String, VarStats> stats )
- {
- if( inst instanceof VariableCPInstruction )
- {
- String optype = inst.getOpcode();
- String[] parts = InstructionUtils.getInstructionParts(inst.toString());
-
- if( optype.equals("createvar") ) {
- if( parts.length<10 )
- return;
- String varname = parts[1];
- long rlen = Long.parseLong(parts[5]);
- long clen = Long.parseLong(parts[6]);
- long brlen = Long.parseLong(parts[7]);
- long bclen = Long.parseLong(parts[8]);
- long nnz = Long.parseLong(parts[9]);
- VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false);
- stats.put(varname, vs);
-
- //System.out.println(varname+" "+vs);
- }
- else if ( optype.equals("cpvar") ) {
- String varname = parts[1];
- String varname2 = parts[2];
- VarStats vs = stats.get(varname);
- stats.put(varname2, vs);
- }
- else if ( optype.equals("mvvar") ) {
- String varname = parts[1];
- String varname2 = parts[2];
- VarStats vs = stats.remove(varname);
- stats.put(varname2, vs);
- }
- else if( optype.equals("rmvar") ) {
- String varname = parts[1];
- stats.remove(varname);
- }
- }
- else if( inst instanceof DataGenCPInstruction ){
- DataGenCPInstruction randInst = (DataGenCPInstruction) inst;
- String varname = randInst.output.getName();
- long rlen = randInst.getRows();
- long clen = randInst.getCols();
- long brlen = randInst.getRowsInBlock();
- long bclen = randInst.getColsInBlock();
- long nnz = (long) (randInst.getSparsity() * rlen * clen);
- VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, true);
- stats.put(varname, vs);
- }
- else if( inst instanceof StringInitCPInstruction ){
- StringInitCPInstruction iinst = (StringInitCPInstruction) inst;
- String varname = iinst.output.getName();
- long rlen = iinst.getRows();
- long clen = iinst.getCols();
- VarStats vs = new VarStats(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, rlen*clen, true);
- stats.put(varname, vs);
- }
- else if( inst instanceof FunctionCallCPInstruction )
- {
- FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
- ArrayList<String> outVars = finst.getBoundOutputParamNames();
- for( String varname : outVars )
- {
- stats.put(varname, _unknownStats);
- //System.out.println(varname+" "+vs);
- }
- }
- }
-
- /**
- *
- * @param inst
- * @param stats
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void maintainMRJobInstVariableStatistics( Instruction inst, HashMap<String, VarStats> stats )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- MRJobInstruction jobinst = (MRJobInstruction)inst;
-
- //input sizes (varname, index mapping)
- String[] inVars = jobinst.getInputVars();
- int index = -1;
- for( String varname : inVars )
- {
- VarStats vs = stats.get(varname);
- if( vs==null )
- vs = _unknownStats;
- stats.put(String.valueOf(++index), vs);
- }
-
- //rand output
- String rdInst = jobinst.getIv_randInstructions();
- if( rdInst != null && rdInst.length()>0 )
- {
- StringTokenizer st = new StringTokenizer(rdInst,Lop.INSTRUCTION_DELIMITOR);
- while( st.hasMoreTokens() ) //foreach rand instruction
- {
- String[] parts = InstructionUtils.getInstructionParts(st.nextToken());
- byte outIndex = Byte.parseByte(parts[2]);
- long rlen = parts[3].contains("##")?-1:UtilFunctions.parseToLong(parts[3]);
- long clen = parts[4].contains("##")?-1:UtilFunctions.parseToLong(parts[4]);
- long brlen = Long.parseLong(parts[5]);
- long bclen = Long.parseLong(parts[6]);
- long nnz = (long) (Double.parseDouble(parts[9]) * rlen * clen);
- VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false);
- stats.put(String.valueOf(outIndex), vs);
- }
- }
-
- //compute intermediate result indices
- HashMap<Byte,MatrixCharacteristics> dims = new HashMap<Byte, MatrixCharacteristics>();
- //populate input indices
- for( Entry<String,VarStats> e : stats.entrySet() )
- {
- if(UtilFunctions.isIntegerNumber(e.getKey()))
- {
- byte ix = Byte.parseByte(e.getKey());
- VarStats vs = e.getValue();
- if( vs !=null )
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(vs._rlen, vs._clen, (int)vs._brlen, (int)vs._bclen, (long)vs._nnz);
- dims.put(ix, mc);
- }
- }
- }
- //compute dims for all instructions
- String[] instCat = new String[]{
- jobinst.getIv_randInstructions(),
- jobinst.getIv_recordReaderInstructions(),
- jobinst.getIv_instructionsInMapper(),
- jobinst.getIv_shuffleInstructions(),
- jobinst.getIv_aggInstructions(),
- jobinst.getIv_otherInstructions()};
- for( String linstCat : instCat )
- if( linstCat !=null && linstCat.length()>0 )
- {
- String[] linst = linstCat.split(Instruction.INSTRUCTION_DELIM);
- for( String instStr : linst )
- {
- String instStr2 = replaceInstructionPatch(instStr);
- MRInstruction mrinst = MRInstructionParser.parseSingleInstruction(instStr2);
- MatrixCharacteristics.computeDimension(dims, mrinst);
- }
- }
-
- //create varstats if necessary
- for( Entry<Byte,MatrixCharacteristics> e : dims.entrySet() )
- {
- byte ix = e.getKey();
- if( !stats.containsKey(String.valueOf(ix)) )
- {
- MatrixCharacteristics mc = e.getValue();
- VarStats vs = new VarStats(mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), mc.getNonZeros(), false);
- stats.put(String.valueOf(ix), vs);
- }
- }
-
- //map result indexes
- String[] outLabels = jobinst.getOutputVars();
- byte[] resultIndexes = jobinst.getIv_resultIndices();
- for( int i=0; i<resultIndexes.length; i++ )
- {
- String varname = outLabels[i];
- VarStats varvs = stats.get(String.valueOf(resultIndexes[i]));
- if( varvs==null )
- {
- varvs = stats.get(outLabels[i]);
- }
- varvs._inmem = false;
- stats.put(varname, varvs);
- }
- }
-
- /**
- *
- * @param inst
- * @return
- */
- protected String replaceInstructionPatch( String inst )
- {
- String ret = inst;
- while( ret.contains("##") ) {
- int index1 = ret.indexOf("##");
- int index2 = ret.indexOf("##", index1+3);
- String replace = ret.substring(index1,index2+2);
- ret = ret.replaceAll(replace, "1");
- }
-
- return ret;
- }
-
- /**
- *
- * @param inst
- * @param stats
- * @return
- */
- private Object[] extractCPInstStatistics( Instruction inst, HashMap<String, VarStats> stats )
- {
- Object[] ret = new Object[2]; //stats, attrs
- VarStats[] vs = new VarStats[3];
- String[] attr = null;
-
- if( inst instanceof UnaryCPInstruction )
- {
- if( inst instanceof DataGenCPInstruction )
- {
- DataGenCPInstruction rinst = (DataGenCPInstruction) inst;
- vs[0] = _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = stats.get( rinst.output.getName() );
-
- //prepare attributes for cost estimation
- int type = 2; //full rand
- if( rinst.getMinValue() == 0.0 && rinst.getMaxValue() == 0.0 )
- type = 0;
- else if( rinst.getSparsity() == 1.0 && rinst.getMinValue() == rinst.getMaxValue() )
- type = 1;
- attr = new String[]{String.valueOf(type)};
- }
- else if( inst instanceof StringInitCPInstruction )
- {
- StringInitCPInstruction rinst = (StringInitCPInstruction) inst;
- vs[0] = _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = stats.get( rinst.output.getName() );
- }
- else //general unary
- {
- UnaryCPInstruction uinst = (UnaryCPInstruction) inst;
- vs[0] = stats.get( uinst.input1.getName() );
- vs[1] = _unknownStats;
- vs[2] = stats.get( uinst.output.getName() );
-
- if( vs[0] == null ) //scalar input, e.g., print
- vs[0] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
-
- if( inst instanceof MMTSJCPInstruction )
- {
- String type = ((MMTSJCPInstruction)inst).getMMTSJType().toString();
- attr = new String[]{type};
- }
- else if( inst instanceof AggregateUnaryCPInstruction )
- {
- String[] parts = InstructionUtils.getInstructionParts(inst.toString());
- String opcode = parts[0];
- if( opcode.equals("cm") )
- attr = new String[]{parts[parts.length-2]};
- }
- }
- }
- else if( inst instanceof BinaryCPInstruction )
- {
- BinaryCPInstruction binst = (BinaryCPInstruction) inst;
- vs[0] = stats.get( binst.input1.getName() );
- vs[1] = stats.get( binst.input2.getName() );
- vs[2] = stats.get( binst.output.getName() );
-
-
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
- }
- else if( inst instanceof AggregateTernaryCPInstruction )
- {
- AggregateTernaryCPInstruction binst = (AggregateTernaryCPInstruction) inst;
- //of same dimension anyway but missing third input
- vs[0] = stats.get( binst.input1.getName() );
- vs[1] = stats.get( binst.input2.getName() );
- vs[2] = stats.get( binst.output.getName() );
-
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
- }
- else if( inst instanceof ParameterizedBuiltinCPInstruction )
- {
- //ParameterizedBuiltinCPInstruction pinst = (ParameterizedBuiltinCPInstruction) inst;
- String[] parts = InstructionUtils.getInstructionParts(inst.toString());
- String opcode = parts[0];
- if( opcode.equals("groupedagg") )
- {
- HashMap<String,String> paramsMap = ParameterizedBuiltinCPInstruction.constructParameterMap(parts);
- String fn = paramsMap.get("fn");
- String order = paramsMap.get("order");
- AggregateOperationTypes type = CMOperator.getAggOpType(fn, order);
- attr = new String[]{String.valueOf(type.ordinal())};
- }
- else if( opcode.equals("rmempty") )
- {
- HashMap<String,String> paramsMap = ParameterizedBuiltinCPInstruction.constructParameterMap(parts);
- attr = new String[]{String.valueOf(paramsMap.get("margin").equals("rows")?0:1)};
- }
-
- vs[0] = stats.get( parts[1].substring(7).replaceAll("##", "") );
- vs[1] = _unknownStats; //TODO
- vs[2] = stats.get( parts[parts.length-1] );
-
- if( vs[0] == null ) //scalar input
- vs[0] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
- }
- else if( inst instanceof MultiReturnBuiltinCPInstruction )
- {
- //applies to qr, lu, eigen (cost computation on input1)
- MultiReturnBuiltinCPInstruction minst = (MultiReturnBuiltinCPInstruction) inst;
- vs[0] = stats.get( minst.input1.getName() );
- vs[1] = stats.get( minst.getOutput(0).getName() );
- vs[2] = stats.get( minst.getOutput(1).getName() );
- }
- else if( inst instanceof VariableCPInstruction )
- {
- setUnknownStats(vs);
-
- VariableCPInstruction varinst = (VariableCPInstruction) inst;
- if( varinst.getOpcode().equals("write") ) {
- //special handling write of matrix objects (non existing if scalar)
- if( stats.containsKey( varinst.getInput1().getName() ) )
- vs[0] = stats.get( varinst.getInput1().getName() );
- attr = new String[]{varinst.getInput3().getName()};
- }
- }
- else
- {
- setUnknownStats(vs);
- }
-
- //maintain var status (CP output always inmem)
- vs[2]._inmem = true;
-
- ret[0] = vs;
- ret[1] = attr;
-
- return ret;
- }
-
- private void setUnknownStats(VarStats[] vs) {
- vs[0] = _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = _unknownStats;
- }
-
- /**
- *
- * @param inst
- * @param stats
- * @return
- */
- private Object[] extractMRJobInstStatistics( Instruction inst, HashMap<String, VarStats> stats )
- {
- Object[] ret = new Object[2]; //stats, attrs
- VarStats[] vs = null;
- String[] attr = null;
-
- MRJobInstruction jinst = (MRJobInstruction)inst;
-
- //get number of indices
- byte[] indexes = jinst.getIv_resultIndices();
- byte maxIx = -1;
- for( int i=0; i<indexes.length; i++ )
- if( maxIx < indexes[i] )
- maxIx = indexes[i];
-
- vs = new VarStats[ maxIx+1 ];
-
- //get inputs, intermediates, and outputs
- for( int i=0; i<vs.length; i++ ){
- vs[i] = stats.get(String.valueOf(i));
- if( vs[i]==null )
- {
- vs[i] = _unknownStats;
- }
- }
-
- //result preparation
- ret[0] = vs;
- ret[1] = attr;
-
- return ret;
- }
-
- /**
- *
- * @param inst
- * @param stats
- */
- private void cleanupMRJobVariableStatistics( Instruction inst, HashMap<String, VarStats> stats )
- {
- MRJobInstruction jinst = (MRJobInstruction)inst;
-
- //get number of indices
- byte[] indexes = jinst.getIv_resultIndices();
- byte maxIx = -1;
- for( int i=0; i<indexes.length; i++ )
- if( maxIx < indexes[i] )
- maxIx = indexes[i];
-
- //remove all stats up to max index
- for( int i=0; i<=maxIx; i++ )
- {
- VarStats tmp = stats.remove(String.valueOf(i));
- if( tmp!=null )
- tmp._inmem = false; //all MR job outptus on HDFS
- }
- }
-
- /**
- * TODO use of vars - needed for recompilation
- * TODO w/o exception
- *
- * @param vars
- * @param stats
- * @param pred
- * @return
- */
- private int getNumIterations(HashMap<String,VarStats> stats, String[] pred)
- {
- int N = DEFAULT_NUMITER;
- if( pred != null && pred[1]!=null && pred[2]!=null && pred[3]!=null )
- {
- try
- {
- int from = Integer.parseInt(pred[1]);
- int to = Integer.parseInt(pred[2]);
- int increment = Integer.parseInt(pred[3]);
- N = (int)Math.ceil(((double)(to-from+1))/increment);
- }
- catch(Exception ex){}
- }
- return N;
- }
-
-
- /**
- *
- * @param inst
- * @param vs
- * @param args
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- protected abstract double getCPInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException;
-
- /**
- *
- * @param inst
- * @param vs
- * @param args
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- protected abstract double getMRJobInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorNumMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorNumMRJobs.java b/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorNumMRJobs.java
deleted file mode 100644
index 4c9e78f..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorNumMRJobs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.cost;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-
-/**
- *
- */
-public class CostEstimatorNumMRJobs extends CostEstimator
-{
-
-
- @Override
- protected double getCPInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- return 0;
- }
-
- @Override
- protected double getMRJobInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorStaticRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorStaticRuntime.java
deleted file mode 100644
index 67d9cb2..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimatorStaticRuntime.java
+++ /dev/null
@@ -1,1317 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.cost;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.LazyWriteBuffer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.IDistributedCacheConsumer;
-import com.ibm.bi.dml.runtime.instructions.mr.MMTSJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MapMultChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PickByCountInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.yarn.ropt.MRJobResourceInstruction;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
-
-/**
- *
- */
-public class CostEstimatorStaticRuntime extends CostEstimator
-{
-
- //time-conversion
- private static final long DEFAULT_FLOPS = 2L * 1024 * 1024 * 1024; //2GFLOPS
- //private static final long UNKNOWN_TIME = -1;
-
- //floating point operations
- private static final double DEFAULT_NFLOP_NOOP = 10;
- private static final double DEFAULT_NFLOP_UNKNOWN = 1;
- private static final double DEFAULT_NFLOP_CP = 1;
- private static final double DEFAULT_NFLOP_TEXT_IO = 350;
-
- //MR job latency
- private static final double DEFAULT_MR_JOB_LATENCY_LOCAL = 2;
- private static final double DEFAULT_MR_JOB_LATENCY_REMOTE = 20;
- private static final double DEFAULT_MR_TASK_LATENCY_LOCAL = 0.001;
- private static final double DEFAULT_MR_TASK_LATENCY_REMOTE = 1.5;
-
- //IO READ throughput
- private static final double DEFAULT_MBS_FSREAD_BINARYBLOCK_DENSE = 200;
- private static final double DEFAULT_MBS_FSREAD_BINARYBLOCK_SPARSE = 100;
- private static final double DEFAULT_MBS_HDFSREAD_BINARYBLOCK_DENSE = 150;
- private static final double DEFAULT_MBS_HDFSREAD_BINARYBLOCK_SPARSE = 75;
- //IO WRITE throughput
- private static final double DEFAULT_MBS_FSWRITE_BINARYBLOCK_DENSE = 150;
- private static final double DEFAULT_MBS_FSWRITE_BINARYBLOCK_SPARSE = 75;
- private static final double DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_DENSE = 120;
- private static final double DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_SPARSE = 60;
- private static final double DEFAULT_MBS_HDFSWRITE_TEXT_DENSE = 40;
- private static final double DEFAULT_MBS_HDFSWRITE_TEXT_SPARSE = 30;
-
- @Override
- @SuppressWarnings("unused")
- protected double getCPInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- CPInstruction cpinst = (CPInstruction)inst;
-
- //load time into mem
- double ltime = 0;
- if( !vs[0]._inmem ){
- ltime += getHDFSReadTime( vs[0]._rlen, vs[0]._clen, vs[0].getSparsity() );
- //eviction costs
- if( CacheableData.CACHING_WRITE_CACHE_ON_READ &&
- LazyWriteBuffer.getWriteBufferSize()<MatrixBlock.estimateSizeOnDisk(vs[0]._rlen, vs[0]._clen, (long)((vs[0]._nnz<0)? vs[0]._rlen*vs[0]._clen:vs[0]._nnz)) )
- {
- ltime += Math.abs( getFSWriteTime( vs[0]._rlen, vs[0]._clen, vs[0].getSparsity() ));
- }
- vs[0]._inmem = true;
- }
- if( !vs[1]._inmem ){
- ltime += getHDFSReadTime( vs[1]._rlen, vs[1]._clen, vs[1].getSparsity() );
- //eviction costs
- if( CacheableData.CACHING_WRITE_CACHE_ON_READ &&
- LazyWriteBuffer.getWriteBufferSize()<MatrixBlock.estimateSizeOnDisk(vs[1]._rlen, vs[1]._clen, (long)((vs[1]._nnz<0)? vs[1]._rlen*vs[1]._clen:vs[1]._nnz)) )
- {
- ltime += Math.abs( getFSWriteTime( vs[1]._rlen, vs[1]._clen, vs[1].getSparsity()) );
- }
- vs[1]._inmem = true;
- }
- if( LOG.isDebugEnabled() && ltime!=0 ) {
- LOG.debug("Cost["+cpinst.getOpcode()+" - read] = "+ltime);
- }
-
- //exec time CP instruction
- String opcode = (cpinst instanceof FunctionCallCPInstruction) ? InstructionUtils.getOpCode(cpinst.toString()) : cpinst.getOpcode();
- double etime = getInstTimeEstimate(opcode, vs, args, ExecType.CP);
-
- //write time caching
- double wtime = 0;
- //double wtime = getFSWriteTime( vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen );
- if( inst instanceof VariableCPInstruction && ((VariableCPInstruction)inst).getOpcode().equals("write") )
- wtime += getHDFSWriteTime(vs[2]._rlen, vs[2]._clen, vs[2].getSparsity(), ((VariableCPInstruction)inst).getInput3().getName() );
-
- if( LOG.isDebugEnabled() && wtime!=0 ) {
- LOG.debug("Cost["+cpinst.getOpcode()+" - write] = "+wtime);
- }
-
- //total costs
- double costs = ltime + etime + wtime;
-
- //if( LOG.isDebugEnabled() )
- // LOG.debug("Costs CP instruction = "+costs);
-
- return costs;
- }
-
-
- @Override
- protected double getMRJobInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- MRJobInstruction jinst = (MRJobInstruction) inst;
-
- //infrastructure properties
- boolean localJob = InfrastructureAnalyzer.isLocalMode();
- int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
- int maxPRed = Math.min( InfrastructureAnalyzer.getRemoteParallelReduceTasks(),
- ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS) );
- double blocksize = ((double)InfrastructureAnalyzer.getHDFSBlockSize())/(1024*1024);
-
- //correction max number of mappers/reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() ) {
- maxPMap = (int)Math.max( maxPMap, YarnClusterAnalyzer.getNumCores() );
- //artificially reduced by factor 2, in order to prefer map-side processing even if smaller degree of parallelism
- maxPRed = (int)Math.max( maxPRed, YarnClusterAnalyzer.getNumCores()/2 /2 );
- }
-
- //yarn-specific: take degree of parallelism into account
- if( jinst instanceof MRJobResourceInstruction ){
- int maxTasks = (int)((MRJobResourceInstruction)jinst).getMaxMRTasks();
- maxPMap = Math.min(maxPMap, maxTasks);
- maxPRed = Math.min(maxPRed, maxTasks);
- }
-
- //job properties
- boolean mapOnly = jinst.isMapOnly();
- String rdInst = jinst.getIv_randInstructions();
- String rrInst = jinst.getIv_recordReaderInstructions();
- String mapInst = jinst.getIv_instructionsInMapper();
- String shfInst = jinst.getIv_shuffleInstructions();
- String aggInst = jinst.getIv_aggInstructions();
- String otherInst = jinst.getIv_otherInstructions();
- byte[] inIx = getInputIndexes( jinst.getInputVars() );
- byte[] retIx = jinst.getIv_resultIndices();
- byte[] mapOutIx = getMapOutputIndexes(inIx, retIx, rdInst, mapInst, shfInst, aggInst, otherInst);
- int numMap = computeNumMapTasks(vs, inIx, blocksize, maxPMap, jinst.getJobType());
- int numPMap = Math.min(numMap, maxPMap);
- int numEPMap = Math.max(Math.min(numMap, maxPMap/2),1); //effective map dop
- int numRed = computeNumReduceTasks( vs, mapOutIx, jinst.getJobType() );
- int numPRed = Math.min(numRed, maxPRed);
- int numEPRed = Math.max(Math.min(numRed, maxPRed/2),1); //effective reduce dop
-
- LOG.debug("Meta nmap = "+numMap+", nred = "+numRed+"; npmap = "+numPMap+", npred = "+numPRed+"; nepmap = "+numEPMap+", nepred = "+numEPRed);
-
- //step 0: export if inputs in mem
- double exportCosts = 0;
- for( int i=0; i<jinst.getInputVars().length; i++ )
- if( vs[i]._inmem )
- exportCosts += getHDFSWriteTime(vs[i]._rlen, vs[i]._clen, vs[i].getSparsity());
-
- //step 1: MR job / task latency (normalization by effective dop)
- double jobLatencyCosts = localJob ? DEFAULT_MR_JOB_LATENCY_LOCAL : DEFAULT_MR_JOB_LATENCY_REMOTE;
- double taskLatencyCost = (numMap / numEPMap + numEPRed)
- * (localJob ? DEFAULT_MR_TASK_LATENCY_LOCAL : DEFAULT_MR_TASK_LATENCY_REMOTE);
- double latencyCosts = jobLatencyCosts + taskLatencyCost;
-
- //step 2: parallel read of inputs (normalization by effective dop)
- double hdfsReadCosts = 0;
- for( int i=0; i<jinst.getInputVars().length; i++ )
- hdfsReadCosts += getHDFSReadTime(vs[i]._rlen, vs[i]._clen, vs[i].getSparsity());
- hdfsReadCosts /= numEPMap;
-
- //step 3: parallel MR instructions
- String[] mapperInst = new String[]{rdInst, rrInst, mapInst};
- String[] reducerInst = new String[]{shfInst, aggInst, otherInst};
-
- //map instructions compute/distcache read (normalization by effective dop)
- double mapDCReadCost = 0; //read through distributed cache
- double mapCosts = 0; //map compute cost
- double shuffleCosts = 0;
- double reduceCosts = 0; //reduce compute costs
-
- for( String instCat : mapperInst )
- if( instCat != null && instCat.length()>0 ) {
- String[] linst = instCat.split( Lop.INSTRUCTION_DELIMITOR );
- for( String tmp : linst ){
- //map compute costs
- Object[] o = extractMRInstStatistics(tmp, vs);
- String opcode = InstructionUtils.getOpCode(tmp);
- mapCosts += getInstTimeEstimate(opcode, (VarStats[])o[0], (String[])o[1], ExecType.MR);
- //dist cache read costs
- int dcIndex = getDistcacheIndex(tmp);
- if( dcIndex >= 0 ) {
- mapDCReadCost += Math.min(getFSReadTime(vs[dcIndex]._rlen, vs[dcIndex]._clen, vs[dcIndex].getSparsity()),
- getFSReadTime(DistributedCacheInput.PARTITION_SIZE, 1, 1.0)) //32MB partitions
- * numMap; //read in each task
- }
- }
- }
- mapCosts /= numEPMap;
- mapDCReadCost /= numEPMap;
-
- if( !mapOnly )
- {
- //shuffle costs (normalization by effective map/reduce dop)
- for( int i=0; i<mapOutIx.length; i++ )
- {
- shuffleCosts += ( getFSWriteTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity()) / numEPMap
- + getFSWriteTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity())*4 / numEPRed
- + getFSReadTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity()) / numEPRed);
-
- //correction of shuffle costs (necessary because the above shuffle does not consider the number of blocks)
- //TODO this is a workaround - we need to address the number of map output blocks in a more systematic way
- for( String instCat : reducerInst )
- if( instCat != null && instCat.length()>0 ) {
- String[] linst = instCat.split( Lop.INSTRUCTION_DELIMITOR );
- for( String tmp : linst ) {
- if(InstructionUtils.getMRType(tmp)==MRINSTRUCTION_TYPE.Aggregate)
- shuffleCosts += numMap * getFSWriteTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity()) / numEPMap
- + numPMap * getFSWriteTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity()) / numEPMap
- + numPMap * getFSReadTime(vs[mapOutIx[i]]._rlen, vs[mapOutIx[i]]._clen, vs[mapOutIx[i]].getSparsity()) / numEPRed;
- }
- }
- }
-
- //reduce instructions compute (normalization by effective dop)
- for( String instCat : reducerInst )
- if( instCat != null && instCat.length()>0 ) {
- String[] linst = instCat.split( Lop.INSTRUCTION_DELIMITOR );
- for( String tmp : linst ){
- Object[] o = extractMRInstStatistics(tmp, vs);
- if(InstructionUtils.getMRType(tmp)==MRINSTRUCTION_TYPE.Aggregate)
- o[1] = new String[]{String.valueOf(numMap)};
- String opcode = InstructionUtils.getOpCode(tmp);
- reduceCosts += getInstTimeEstimate(opcode, (VarStats[])o[0], (String[])o[1], ExecType.MR);
- }
- }
- reduceCosts /= numEPRed;
- }
-
- //step 4: parallel write of outputs (normalization by effective dop)
- double hdfsWriteCosts = 0;
- for( int i=0; i<jinst.getOutputVars().length; i++ )
- {
- hdfsWriteCosts += getHDFSWriteTime(vs[retIx[i]]._rlen, vs[retIx[i]]._clen, vs[retIx[i]].getSparsity());
- }
- hdfsWriteCosts /= ((mapOnly)? numEPMap : numEPRed);
-
- //debug output
- if( LOG.isDebugEnabled() ) {
- LOG.debug("Costs Export = "+exportCosts);
- LOG.debug("Costs Latency = "+latencyCosts);
- LOG.debug("Costs HDFS Read = "+hdfsReadCosts);
- LOG.debug("Costs Distcache Read = "+mapDCReadCost);
- LOG.debug("Costs Map Exec = "+mapCosts);
- LOG.debug("Costs Shuffle = "+shuffleCosts);
- LOG.debug("Costs Reduce Exec = "+reduceCosts);
- LOG.debug("Costs HDFS Write = "+hdfsWriteCosts);
- }
-
- //aggregate individual cost factors
- return exportCosts + latencyCosts +
- hdfsReadCosts + mapCosts + mapDCReadCost +
- shuffleCosts +
- reduceCosts + hdfsWriteCosts;
- }
-
- /**
- *
- * @param inst
- * @param stats
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- private Object[] extractMRInstStatistics( String inst, VarStats[] stats )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- Object[] ret = new Object[2]; //stats, attrs
- VarStats[] vs = new VarStats[3];
- String[] attr = null;
-
- String[] parts = InstructionUtils.getInstructionParts(inst);
- String opcode = parts[0];
-
-
- if( opcode.equals(DataGen.RAND_OPCODE) )
- {
- vs[0] = _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = stats[Integer.parseInt(parts[2])];
-
- int type = 2;
- //awareness of instruction patching min/max
- if( !parts[7].contains(Lop.VARIABLE_NAME_PLACEHOLDER)
- && !parts[8].contains(Lop.VARIABLE_NAME_PLACEHOLDER) )
- {
- double minValue = Double.parseDouble(parts[7]);
- double maxValue = Double.parseDouble(parts[8]);
- double sparsity = Double.parseDouble(parts[9]);
- if( minValue == 0.0 && maxValue == 0.0 )
- type = 0;
- else if( sparsity == 1.0 && minValue == maxValue )
- type = 1;
- }
- attr = new String[]{String.valueOf(type)};
- }
- if( opcode.equals(DataGen.SEQ_OPCODE) )
- {
- vs[0] = _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = stats[Integer.parseInt(parts[2])];
- }
- else //general case
- {
-
- String inst2 = replaceInstructionPatch( inst );
- MRInstruction mrinst = MRInstructionParser.parseSingleInstruction(inst2);
-
- if( mrinst instanceof UnaryMRInstructionBase )
- {
- UnaryMRInstructionBase uinst = (UnaryMRInstructionBase) mrinst;
- vs[0] = uinst.input>=0 ? stats[ uinst.input ] : _unknownStats;
- vs[1] = _unknownStats;
- vs[2] = stats[ uinst.output ];
-
- if( vs[0] == null ) //scalar input, e.g., print
- vs[0] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
-
- if( mrinst instanceof MMTSJMRInstruction )
- {
- String type = ((MMTSJMRInstruction)mrinst).getMMTSJType().toString();
- attr = new String[]{type};
- }
- else if( mrinst instanceof CM_N_COVInstruction )
- {
- if( opcode.equals("cm") )
- attr = new String[]{parts[parts.length-2]};
- }
- else if( mrinst instanceof GroupedAggregateInstruction )
- {
- if( opcode.equals("groupedagg") )
- {
- AggregateOperationTypes type = CMOperator.getAggOpType(parts[2], parts[3]);
- attr = new String[]{String.valueOf(type.ordinal())};
- }
-
- }
- }
- else if( mrinst instanceof BinaryMRInstructionBase )
- {
- BinaryMRInstructionBase binst = (BinaryMRInstructionBase) mrinst;
- vs[0] = stats[ binst.input1 ];
- vs[1] = stats[ binst.input2 ];
- vs[2] = stats[ binst.output ];
-
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar output
- vs[2] = _scalarStats;
-
- if( opcode.equals("rmempty") ) {
- RemoveEmptyMRInstruction rbinst = (RemoveEmptyMRInstruction) mrinst;
- attr = new String[]{rbinst.isRemoveRows()?"0":"1"};
- }
- }
- else if( mrinst instanceof TernaryInstruction )
- {
- TernaryInstruction tinst = (TernaryInstruction) mrinst;
- vs[0] = stats[ tinst.input1 ];
- vs[1] = stats[ tinst.input2 ];
- vs[2] = stats[ tinst.input3 ];
-
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar input
- vs[2] = _scalarStats;
- }
- else if( mrinst instanceof PickByCountInstruction )
- {
- PickByCountInstruction pinst = (PickByCountInstruction) mrinst;
- vs[0] = stats[ pinst.input1 ];
- vs[2] = stats[ pinst.output ];
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar input
- vs[2] = _scalarStats;
- }
- else if( mrinst instanceof MapMultChainInstruction)
- {
- MapMultChainInstruction minst = (MapMultChainInstruction) mrinst;
- vs[0] = stats[ minst.getInput1() ];
- vs[1] = stats[ minst.getInput2() ];
- if( minst.getInput3()>=0 )
- vs[2] = stats[ minst.getInput3() ];
-
- if( vs[0] == null ) //scalar input,
- vs[0] = _scalarStats;
- if( vs[1] == null ) //scalar input,
- vs[1] = _scalarStats;
- if( vs[2] == null ) //scalar input
- vs[2] = _scalarStats;
- }
- }
-
- //maintain var status (CP output always inmem)
- vs[2]._inmem = true;
-
- ret[0] = vs;
- ret[1] = attr;
-
- return ret;
- }
-
-
-
- /////////////////////
- // Utilities //
- /////////////////////
-
- /**
- *
- * @param inputVars
- * @return
- */
- private byte[] getInputIndexes(String[] inputVars)
- {
- byte[] inIx = new byte[inputVars.length];
- for( int i=0; i<inIx.length; i++ )
- inIx[i] = (byte)i;
- return inIx;
- }
-
- /**
- *
- * @param inIx
- * @param retIx
- * @param rdInst
- * @param mapInst
- * @param shfInst
- * @param aggInst
- * @param otherInst
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- private byte[] getMapOutputIndexes( byte[] inIx, byte[] retIx, String rdInst, String mapInst, String shfInst, String aggInst, String otherInst )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //note: this is a simplified version of MRJobConfiguration.setUpOutputIndexesForMapper
-
- //map indices
- HashSet<Byte> ixMap = new HashSet<Byte>();
- for( byte ix : inIx )
- ixMap.add(ix);
-
- if( rdInst!=null && rdInst.length()>0 ) {
- rdInst = replaceInstructionPatch(rdInst);
- DataGenMRInstruction[] ins = MRInstructionParser.parseDataGenInstructions(rdInst);
- for( DataGenMRInstruction inst : ins )
- for( byte ix : inst.getAllIndexes() )
- ixMap.add(ix);
- }
-
- if( mapInst!=null && mapInst.length()>0 ) {
- mapInst = replaceInstructionPatch(mapInst);
- MRInstruction[] ins = MRInstructionParser.parseMixedInstructions(mapInst);
- for( MRInstruction inst : ins )
- for( byte ix : inst.getAllIndexes() )
- ixMap.add(ix);
- }
-
- //reduce indices
- HashSet<Byte> ixRed = new HashSet<Byte>();
- for( byte ix : retIx )
- ixRed.add(ix);
-
-
- if( shfInst!=null && shfInst.length()>0 ) {
- shfInst = replaceInstructionPatch(shfInst);
- MRInstruction[] ins = MRInstructionParser.parseMixedInstructions(shfInst);
- for( MRInstruction inst : ins )
- for( byte ix : inst.getAllIndexes() )
- ixRed.add(ix);
- }
-
- if( aggInst!=null && aggInst.length()>0 ) {
- aggInst = replaceInstructionPatch(aggInst);
- MRInstruction[] ins = MRInstructionParser.parseAggregateInstructions(aggInst);
- for( MRInstruction inst : ins )
- for( byte ix : inst.getAllIndexes() )
- ixRed.add(ix);
- }
-
- if( otherInst!=null && otherInst.length()>0 ) {
- otherInst = replaceInstructionPatch(otherInst);
- MRInstruction[] ins = MRInstructionParser.parseMixedInstructions(otherInst);
- for( MRInstruction inst : ins )
- for( byte ix : inst.getAllIndexes() )
- ixRed.add(ix);
- }
-
- //difference
- ixMap.retainAll(ixRed);
-
- //copy result
- byte[] ret = new byte[ixMap.size()];
- int i = 0;
- for( byte ix : ixMap )
- ret[i++] = ix;
-
- return ret;
- }
-
- /**
- *
- * @param vs
- * @param inputIx
- * @param blocksize
- * @param maxPMap
- * @param jobtype
- * @return
- */
- private int computeNumMapTasks( VarStats[] vs, byte[] inputIx, double blocksize, int maxPMap, JobType jobtype )
- {
- //special cases
- if( jobtype == JobType.DATAGEN )
- return maxPMap;
-
- //input size, num blocks
- double mapInputSize = 0;
- int numBlocks = 0;
- for( int i=0; i<inputIx.length; i++ )
- {
- //input size
- mapInputSize += ((double)MatrixBlock.estimateSizeOnDisk((long)vs[inputIx[i]]._rlen, (long)vs[inputIx[i]]._clen, (long)vs[inputIx[i]]._nnz)) / (1024*1024);
-
- //num blocks
- int lret = (int) Math.ceil((double)vs[inputIx[i]]._rlen/vs[inputIx[i]]._brlen)
- *(int) Math.ceil((double)vs[inputIx[i]]._clen/vs[inputIx[i]]._bclen);
- numBlocks = Math.max(lret, numBlocks);
- }
-
- return Math.max(1, Math.min( (int)Math.ceil(mapInputSize/blocksize),numBlocks ));
- }
-
- /**
- *
- * @param vs
- * @param mapOutIx
- * @param jobtype
- * @return
- */
- private int computeNumReduceTasks( VarStats[] vs, byte[] mapOutIx, JobType jobtype )
- {
- int ret = -1;
-
- //TODO for jobtype==JobType.MMCJ common dim
-
- switch( jobtype )
- {
- case REBLOCK:
- case CSV_REBLOCK: {
- for( int i=0; i<mapOutIx.length; i++ )
- {
- int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/vs[mapOutIx[i]]._brlen)
- *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/vs[mapOutIx[i]]._bclen);
- ret = Math.max(lret, ret);
- }
- break;
- }
-
- default: {
- for( int i=0; i<mapOutIx.length; i++ )
- {
- int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/DMLTranslator.DMLBlockSize)
- *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/DMLTranslator.DMLBlockSize);
- ret = Math.max(lret, ret);
- }
- break;
- }
- }
-
- return Math.max(1, ret);
- }
-
- /**
- *
- * @param inst
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private int getDistcacheIndex(String inst)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- ArrayList<Byte> indexes = new ArrayList<Byte>();
-
- if( InstructionUtils.isDistributedCacheUsed(inst) ) {
- MRInstruction mrinst = MRInstructionParser.parseSingleInstruction(inst);
- if( mrinst instanceof IDistributedCacheConsumer )
- ((IDistributedCacheConsumer)mrinst).addDistCacheIndex(inst, indexes);
- }
-
- if( !indexes.isEmpty() )
- return indexes.get(0);
- else
- return -1;
- }
-
-
- /////////////////////
- // I/O Costs //
- /////////////////////
-
- /**
- * Returns the estimated read time from HDFS.
- * NOTE: Does not handle unknowns.
- *
- * @param dm
- * @param dn
- * @param ds
- * @return
- */
- private double getHDFSReadTime( long dm, long dn, double ds )
- {
- boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn));
- double ret = ((double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn))) / (1024*1024);
-
- if( sparse )
- ret /= DEFAULT_MBS_HDFSREAD_BINARYBLOCK_SPARSE;
- else //dense
- ret /= DEFAULT_MBS_HDFSREAD_BINARYBLOCK_DENSE;
-
- return ret;
- }
-
- /**
- *
- * @param dm
- * @param dn
- * @param ds
- * @return
- */
- private double getHDFSWriteTime( long dm, long dn, double ds )
- {
- boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn));
-
- double bytes = (double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn));
- double mbytes = bytes / (1024*1024);
-
- double ret = -1;
- if( sparse )
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_SPARSE;
- else //dense
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_DENSE;
-
- //if( LOG.isDebugEnabled() )
- // LOG.debug("Costs[export] = "+ret+"s, "+mbytes+" MB ("+dm+","+dn+","+ds+").");
-
-
- return ret;
- }
-
- private double getHDFSWriteTime( long dm, long dn, double ds, String format )
- {
- boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn));
-
- double bytes = (double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn));
- double mbytes = bytes / (1024*1024);
-
- double ret = -1;
-
- if( format.equals("textcell") || format.equals("csv") )
- {
- if( sparse )
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_TEXT_SPARSE;
- else //dense
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_TEXT_DENSE;
- ret *= 2.75; //text commonly 2x-3.5x larger than binary
- }
- else
- {
- if( sparse )
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_SPARSE;
- else //dense
- ret = mbytes / DEFAULT_MBS_HDFSWRITE_BINARYBLOCK_DENSE;
- }
- //if( LOG.isDebugEnabled() )
- // LOG.debug("Costs[export] = "+ret+"s, "+mbytes+" MB ("+dm+","+dn+","+ds+").");
-
-
- return ret;
- }
-
- /**
- * Returns the estimated read time from local FS.
- * NOTE: Does not handle unknowns.
- *
- * @param dm
- * @param dn
- * @param ds
- * @return
- */
- private double getFSReadTime( long dm, long dn, double ds )
- {
- boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn));
-
- double ret = ((double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn))) / (1024*1024);
- if( sparse )
- ret /= DEFAULT_MBS_FSREAD_BINARYBLOCK_SPARSE;
- else //dense
- ret /= DEFAULT_MBS_FSREAD_BINARYBLOCK_DENSE;
-
- return ret;
- }
-
- /**
- *
- * @param dm
- * @param dn
- * @param ds
- * @return
- */
- private double getFSWriteTime( long dm, long dn, double ds )
- {
- boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn));
-
- double ret = ((double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn))) / (1024*1024);
-
- if( sparse )
- ret /= DEFAULT_MBS_FSWRITE_BINARYBLOCK_SPARSE;
- else //dense
- ret /= DEFAULT_MBS_FSWRITE_BINARYBLOCK_DENSE;
-
- return ret;
- }
-
-
- /////////////////////
- // Operation Costs //
- /////////////////////
-
- /**
- *
- * @param inst
- * @param vs
- * @param args
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private double getInstTimeEstimate(String opcode, VarStats[] vs, String[] args, ExecType et)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- boolean inMR = (et == ExecType.MR);
- return getInstTimeEstimate(opcode, inMR,
- vs[0]._rlen, vs[0]._clen, (vs[0]._nnz<0)? 1.0:(double)vs[0]._nnz/vs[0]._rlen/vs[0]._clen,
- vs[1]._rlen, vs[1]._clen, (vs[1]._nnz<0)? 1.0:(double)vs[1]._nnz/vs[1]._rlen/vs[1]._clen,
- vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen,
- args);
- }
-
- /**
- * Returns the estimated instruction execution time, w/o data transfer and single-threaded.
- * For scalars input dims must be set to 1 before invocation.
- *
- * NOTE: Does not handle unknowns.
- *
- * @param opcode
- * @param d1m
- * @param d1n
- * @param d1s
- * @param d2m
- * @param d2n
- * @param d2s
- * @param d3m
- * @param d3n
- * @param d3s
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private double getInstTimeEstimate( String opcode, boolean inMR, long d1m, long d1n, double d1s, long d2m, long d2n, double d2s, long d3m, long d3n, double d3s, String[] args ) throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- double nflops = getNFLOP(opcode, inMR, d1m, d1n, d1s, d2m, d2n, d2s, d3m, d3n, d3s, args);
- double time = nflops / DEFAULT_FLOPS;
-
- if( LOG.isDebugEnabled() )
- LOG.debug("Cost["+opcode+"] = "+time+"s, "+nflops+" flops ("+d1m+","+d1n+","+d1s+","+d2m+","+d2n+","+d2s+","+d3m+","+d3n+","+d3s+").");
-
- return time;
- }
-
- /**
- *
- * @param optype
- * @param d1m
- * @param d1n
- * @param d1s
- * @param d2m
- * @param d2n
- * @param d2s
- * @param d3m
- * @param d3n
- * @param d3s
- * @param args
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private double getNFLOP( String optype, boolean inMR, long d1m, long d1n, double d1s, long d2m, long d2n, double d2s, long d3m, long d3n, double d3s, String[] args )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //operation costs in FLOP on matrix block level (for CP and MR instructions)
- //(excludes IO and parallelism; assumes known dims for all inputs, outputs )
-
- boolean leftSparse = MatrixBlock.evalSparseFormatInMemory(d1m, d1n, (long)(d1s*d1m*d1n));
- boolean rightSparse = MatrixBlock.evalSparseFormatInMemory(d2m, d2n, (long)(d2s*d2m*d2n));
- boolean onlyLeft = (d1m>=0 && d1n>=0 && d2m<0 && d2n<0 );
- boolean allExists = (d1m>=0 && d1n>=0 && d2m>=0 && d2n>=0 && d3m>=0 && d3n>=0 );
-
- //NOTE: all instruction types that are equivalent in CP and MR are only
- //included in CP to prevent redundancy
- CPINSTRUCTION_TYPE cptype = CPInstructionParser.String2CPInstructionType.get(optype);
- if( cptype != null ) //for CP Ops and equivalent MR ops
- {
- //general approach: count of floating point *, /, +, -, ^, builtin ;
- switch(cptype)
- {
-
- case AggregateBinary: //opcodes: ba+*, cov
- if( optype.equals("ba+*") ) { //matrix mult
- //reduction by factor 2 because matrix mult better than
- //average flop count
- if( !leftSparse && !rightSparse )
- return 2 * (d1m * d1n * ((d2n>1)?d1s:1.0) * d2n) /2;
- else if( !leftSparse && rightSparse )
- return 2 * (d1m * d1n * d1s * d2n * d2s) /2;
- else if( leftSparse && !rightSparse )
- return 2 * (d1m * d1n * d1s * d2n) /2;
- else //leftSparse && rightSparse
- return 2 * (d1m * d1n * d1s * d2n * d2s) /2;
- }
- else if( optype.equals("cov") ) {
- //note: output always scalar, d3 used as weights block
- //if( allExists ), same runtime for 2 and 3 inputs
- return 23 * d1m; //(11+3*k+)
- }
-
- return 0;
-
- case MMChain:
- //reduction by factor 2 because matrix mult better than average flop count
- //(mmchain essentially two matrix-vector muliplications)
- if( !leftSparse )
- return (2+2) * (d1m * d1n) /2;
- else
- return (2+2) * (d1m * d1n * d1s) /2;
-
- case AggregateTernary: //opcodes: tak+*
- return 6 * d1m * d1n; //2*1(*) + 4 (k+)
-
- case AggregateUnary: //opcodes: uak+, uark+, uack+, uasqk+, uarsqk+, uacsqk+,
- // uamean, uarmean, uacmean,
- // uamax, uarmax, uarimax, uacmax, uamin, uarmin, uacmin,
- // ua+, uar+, uac+, ua*, uatrace, uaktrace,
- // nrow, ncol, length, cm
-
- if( optype.equals("nrow") || optype.equals("ncol") || optype.equals("length") )
- return DEFAULT_NFLOP_NOOP;
- else if( optype.equals( "cm" ) ) {
- double xcm = 1;
- switch( Integer.parseInt(args[0]) ) {
- case 0: xcm=1; break; //count
- case 1: xcm=8; break; //mean
- case 2: xcm=16; break; //cm2
- case 3: xcm=31; break; //cm3
- case 4: xcm=51; break; //cm4
- case 5: xcm=16; break; //variance
- }
- return (leftSparse) ? xcm * (d1m * d1s + 1) : xcm * d1m;
- }
- else if( optype.equals("uatrace") || optype.equals("uaktrace") )
- return 2 * d1m * d1n;
- else if( optype.equals("ua+") || optype.equals("uar+") || optype.equals("uar+") ){
- //sparse safe operations
- if( !leftSparse ) //dense
- return d1m * d1n;
- else //sparse
- return d1m * d1n * d1s;
- }
- else if( optype.equals("uak+") || optype.equals("uark+") || optype.equals("uark+"))
- return 4 * d1m * d1n; //1*k+
- else if( optype.equals("uasqk+") || optype.equals("uarsqk+") || optype.equals("uacsqk+"))
- return 5 * d1m * d1n; // +1 for multiplication to square term
- else if( optype.equals("uamean") || optype.equals("uarmean") || optype.equals("uacmean"))
- return 7 * d1m * d1n; //1*k+
- else if( optype.equals("uamax") || optype.equals("uarmax") || optype.equals("uacmax")
- || optype.equals("uamin") || optype.equals("uarmin") || optype.equals("uacmin")
- || optype.equals("uarimax") || optype.equals("ua*") )
- return d1m * d1n;
-
- return 0;
-
- case ArithmeticBinary: //opcodes: +, -, *, /, ^ (incl. ^2, *2)
- //note: covers scalar-scalar, scalar-matrix, matrix-matrix
- if( optype.equals("+") || optype.equals("-") //sparse safe
- && ( leftSparse || rightSparse ) )
- return d1m*d1n*d1s + d2m*d2n*d2s;
- else
- return d3m*d3n;
-
- case Ternary: //opcodes: ctable
- if( optype.equals("ctable") ){
- if( leftSparse )
- return d1m * d1n * d1s; //add
- else
- return d1m * d1n;
- }
- return 0;
-
- case BooleanBinary: //opcodes: &&, ||
- return 1; //always scalar-scalar
-
- case BooleanUnary: //opcodes: !
- return 1; //always scalar-scalar
-
- case Builtin: //opcodes: log
- //note: covers scalar-scalar, scalar-matrix, matrix-matrix
- //note: can be unary or binary
- if( allExists ) //binary
- return 3 * d3m * d3n;
- else //unary
- return d3m * d3n;
-
- case BuiltinBinary: //opcodes: max, min, solve
- //note: covers scalar-scalar, scalar-matrix, matrix-matrix
- if( optype.equals("solve") ) //see also MultiReturnBuiltin
- return d1m * d1n * d1n; //for 1kx1k ~ 1GFLOP -> 0.5s
- else //default
- return d3m * d3n;
-
-
- case BuiltinUnary: //opcodes: exp, abs, sin, cos, tan, sqrt, plogp, print, round, sprop, sigmoid
- if( optype.equals("print") ) //scalar only
- return 1;
- else
- {
- double xbu = 1; //default for all ops
- if( optype.equals("plogp") ) xbu = 2;
- else if( optype.equals("round") ) xbu = 4;
-
- if( optype.equals("sin") || optype.equals("tan") || optype.equals("round")
- || optype.equals("abs") || optype.equals("sqrt") || optype.equals("sprop")
- || optype.equals("sigmoid") ) //sparse-safe
- {
- if( leftSparse ) //sparse
- return xbu * d1m * d1n * d1s;
- else //dense
- return xbu * d1m * d1n;
- }
- else
- return xbu * d1m * d1n;
- }
-
- case Reorg: //opcodes: r', rdiag
- case MatrixReshape: //opcodes: rshape
- if( leftSparse )
- return d1m * d1n * d1s;
- else
- return d1m * d1n;
-
- case Append: //opcodes: append
- return DEFAULT_NFLOP_CP *
- (((leftSparse) ? d1m * d1n * d1s : d1m * d1n ) +
- ((rightSparse) ? d2m * d2n * d2s : d2m * d2n ));
-
- case RelationalBinary: //opcodes: ==, !=, <, >, <=, >=
- //note: all relational ops are not sparsesafe
- return d3m * d3n; //covers all combinations of scalar and matrix
-
- case File: //opcodes: rm, mv
- return DEFAULT_NFLOP_NOOP;
-
- case Variable: //opcodes: assignvar, cpvar, rmvar, rmfilevar, assignvarwithfile, attachfiletovar, valuepick, iqsize, read, write, createvar, setfilename, castAsMatrix
- if( optype.equals("write") ){
- boolean text = args[0].equals("textcell") || args[0].equals("csv");
- double xwrite = text ? DEFAULT_NFLOP_TEXT_IO : DEFAULT_NFLOP_CP;
-
- if( !leftSparse )
- return d1m * d1n * xwrite;
- else
- return d1m * d1n * d1s * xwrite;
- }
- else if ( optype.equals("inmem-iqm") )
- //note: assumes uniform distribution
- return 2 * d1m + //sum of weights
- 5 + 0.25d * d1m + //scan to lower quantile
- 8 * 0.5 * d1m; //scan from lower to upper quantile
- else
- return DEFAULT_NFLOP_NOOP;
-
- case Rand: //opcodes: rand, seq
- if( optype.equals(DataGen.RAND_OPCODE) ){
- int nflopRand = 32; //per random number
- switch(Integer.parseInt(args[0])) {
- case 0: return DEFAULT_NFLOP_NOOP; //empty matrix
- case 1: return d3m * d3n * 8; //allocate, arrayfill
- case 2: //full rand
- {
- if( d3s==1.0 )
- return d3m * d3n * nflopRand + d3m * d3n * 8; //DENSE gen (incl allocate)
- else
- return (d3s>=MatrixBlock.SPARSITY_TURN_POINT)?
- 2 * d3m * d3n * nflopRand + d3m * d3n * 8: //DENSE gen (incl allocate)
- 3 * d3m * d3n * d3s * nflopRand + d3m * d3n * d3s * 24; //SPARSE gen (incl allocate)
- }
- }
- }
- else //seq
- return d3m * d3n * DEFAULT_NFLOP_CP;
-
- case StringInit: //sinit
- return d3m * d3n * DEFAULT_NFLOP_CP;
-
- case External: //opcodes: extfunct
- //note: should be invoked independently for multiple outputs
- return d1m * d1n * d1s * DEFAULT_NFLOP_UNKNOWN;
-
- case MultiReturnBuiltin: //opcodes: qr, lu, eigen
- //note: they all have cubic complexity, the scaling factor refers to commons.math
- double xf = 2; //default e.g, qr
- if( optype.equals("eigen") )
- xf = 32;
- else if ( optype.equals("lu") )
- xf = 16;
- return xf * d1m * d1n * d1n; //for 1kx1k ~ 2GFLOP -> 1s
-
- case ParameterizedBuiltin: //opcodes: cdf, invcdf, groupedagg, rmempty
- if( optype.equals("cdf") || optype.equals("invcdf"))
- return DEFAULT_NFLOP_UNKNOWN; //scalar call to commons.math
- else if( optype.equals("groupedagg") ){
- double xga = 1;
- switch( Integer.parseInt(args[0]) ) {
- case 0: xga=4; break; //sum, see uk+
- case 1: xga=1; break; //count, see cm
- case 2: xga=8; break; //mean
- case 3: xga=16; break; //cm2
- case 4: xga=31; break; //cm3
- case 5: xga=51; break; //cm4
- case 6: xga=16; break; //variance
- }
- return 2 * d1m + xga * d1m; //scan for min/max, groupedagg
- }
- else if( optype.equals("rmempty") ){
- switch(Integer.parseInt(args[0])){
- case 0: //remove rows
- return ((leftSparse) ? d1m : d1m * Math.ceil(1.0d/d1s)/2) +
- DEFAULT_NFLOP_CP * d3m * d2m;
- case 1: //remove cols
- return d1n * Math.ceil(1.0d/d1s)/2 +
- DEFAULT_NFLOP_CP * d3m * d2m;
- }
-
- }
- return 0;
-
- case QSort: //opcodes: sort
- if( optype.equals("sort") ){
- //note: mergesort since comparator used
- double sortCosts = 0;
- if( onlyLeft )
- sortCosts = DEFAULT_NFLOP_CP * d1m + d1m;
- else //w/ weights
- sortCosts = DEFAULT_NFLOP_CP * ((leftSparse)?d1m*d1s:d1m);
- return sortCosts + d1m*(int)(Math.log(d1m)/Math.log(2)) + //mergesort
- DEFAULT_NFLOP_CP * d1m;
- }
- return 0;
-
- case MatrixIndexing: //opcodes: rangeReIndex, leftIndex
- if( optype.equals("leftIndex") ){
- return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n)
- + 2 * DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n );
- }
- else if( optype.equals("rangeReIndex") ){
- return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n );
- }
- return 0;
-
- case MMTSJ: //opcodes: tsmm
- //diff to ba+* only upper triangular matrix
- //reduction by factor 2 because matrix mult better than
- //average flop count
- if( MMTSJType.valueOf(args[0]).isLeft() ) { //lefttranspose
- if( !rightSparse ) //dense
- return d1m * d1n * d1s * d1n /2;
- else //sparse
- return d1m * d1n * d1s * d1n * d1s /2;
- }
- else if(onlyLeft) { //righttranspose
- if( !leftSparse ) //dense
- return (double)d1m * d1n * d1m /2;
- else //sparse
- return d1m * d1n * d1s //reorg sparse
- + d1m * d1n * d1s * d1n * d1s /2; //core tsmm
- }
- return 0;
-
- case Partition:
- return d1m * d1n * d1s + //partitioning costs
- (inMR ? 0 : //include write cost if in CP
- getHDFSWriteTime(d1m, d1n, d1s)* DEFAULT_FLOPS);
-
- case INVALID:
- return 0;
-
- default:
- throw new DMLRuntimeException("CostEstimator: unsupported instruction type: "+optype);
- }
-
- }
-
- //if not found in CP instructions
- MRINSTRUCTION_TYPE mrtype = MRInstructionParser.String2MRInstructionType.get(optype);
- if ( mrtype != null ) //for specific MR ops
- {
- switch(mrtype)
- {
- case Aggregate: //opcodes: a+, ak+, asqk+, a*, amax, amin, amean
- //TODO should be aggregate unary
- int numMap = Integer.parseInt(args[0]);
- if( optype.equals("ak+") )
- return 4 * numMap * d1m * d1n * d1s;
- else if( optype.equals("asqk+") )
- return 5 * numMap * d1m * d1n * d1s; // +1 for multiplication to square term
- else
- return numMap * d1m * d1n * d1s;
-
- case AggregateBinary: //opcodes: cpmm, rmm, mapmult
- //note: copy from CP costs
- if( optype.equals("cpmm") || optype.equals("rmm")
- || optype.equals(MapMult.OPCODE) ) //matrix mult
- {
- //reduction by factor 2 because matrix mult better than
- //average flop count
- if( !leftSparse && !rightSparse )
- return 2 * (d1m * d1n * ((d2n>1)?d1s:1.0) * d2n) /2;
- else if( !leftSparse && rightSparse )
- return 2 * (d1m * d1n * d1s * d2n * d2s) /2;
- else if( leftSparse && !rightSparse )
- return 2 * (d1m * d1n * d1s * d2n) /2;
- else //leftSparse && rightSparse
- return 2 * (d1m * d1n * d1s * d2n * d2s) /2;
- }
- return 0;
-
- case MapMultChain: //opcodes: mapmultchain
- //assume dense input2 and input3
- return 2 * d1m * d2n * d1n * ((d2n>1)?d1s:1.0) //ba(+*)
- + d1m * d2n //cellwise b(*)
- + d1m * d2n //r(t)
- + 2 * d2n * d1n * d1m * (leftSparse?d1s:1.0) //ba(+*)
- + d2n * d1n; //r(t)
-
- case ArithmeticBinary: //opcodes: s-r, so, max, min,
- // >, >=, <, <=, ==, !=
- //TODO Should be relational
-
- //note: all relational ops are not sparsesafe
- return d3m * d3n; //covers all combinations of scalar and matrix
-
- case CombineUnary: //opcodes: combineunary
- return d1m * d1n * d1s;
-
- case CombineBinary: //opcodes: combinebinary
- return d1m * d1n * d1s
- + d2m * d2n * d2s;
-
- case CombineTernary: //opcodes: combinetertiary
- return d1m * d1n * d1s
- + d2m * d2n * d2s
- + d3m * d3n * d3s;
-
- case Unary: //opcodes: log, slog, pow
- //TODO requires opcode consolidation (builtin, arithmic)
- //note: covers scalar, matrix, matrix-scalar
- return d3m * d3n;
-
- case Ternary: //opcodes: ctabletransform, ctabletransformscalarweight, ctabletransformhistogram, ctabletransformweightedhistogram
- //note: copy from cp
- if( leftSparse )
- return d1m * d1n * d1s; //add
- else
- return d1m * d1n;
-
- case Quaternary:
- //TODO pattern specific and all 4 inputs requires
- return d1m * d1n * d1s *4;
-
- case Reblock: //opcodes: rblk
- return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n);
-
- case Replicate: //opcodes: rblk
- return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n);
-
- case CM_N_COV: //opcodes: mean
- double xcm = 8;
- return (leftSparse) ? xcm * (d1m * d1s + 1) : xcm * d1m;
-
- case GroupedAggregate: //opcodes: groupedagg
- //TODO: need to consolidate categories (ParameterizedBuiltin)
- //copy from CP opertion
- double xga = 1;
- switch( Integer.parseInt(args[0]) ) {
- case 0: xga=4; break; //sum, see uk+
- case 1: xga=1; break; //count, see cm
- case 2: xga=8; break; //mean
- case 3: xga=16; break; //cm2
- case 4: xga=31; break; //cm3
- case 5: xga=51; break; //cm4
- case 6: xga=16; break; //variance
- }
- return 2 * d1m + xga * d1m; //scan for min/max, groupedagg
-
- case PickByCount: //opcodes: valuepick, rangepick
- break;
- //TODO
- //String2MRInstructionType.put( "valuepick" , MRINSTRUCTION_TYPE.PickByCount); // for quantile()
- //String2MRInstructionType.put( "rangepick" , MRINSTRUCTION_TYPE.PickByCount); // for interQuantile()
-
- case RangeReIndex: //opcodes: rangeReIndex, rangeReIndexForLeft
- //TODO: requires category consolidation
- if( optype.equals("rangeReIndex") )
- return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n );
- else //rangeReIndexForLeft
- return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n)
- + DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n );
-
- case ZeroOut: //opcodes: zeroOut
- return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n)
- + DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n );
-
- default:
- return 0;
- }
- }
- else
- {
- throw new DMLRuntimeException("CostEstimator: unsupported instruction type: "+optype);
- }
-
- //TODO Parameterized Builtin Functions
- //String2CPFileInstructionType.put( "rmempty" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
-
- return -1; //should never come here.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/cost/VarStats.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/cost/VarStats.java b/src/main/java/com/ibm/bi/dml/hops/cost/VarStats.java
deleted file mode 100644
index b1d9671..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/cost/VarStats.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.cost;
-
-/**
- *
- *
- */
-public class VarStats
-{
-
-
- long _rlen = -1;
- long _clen = -1;
- long _brlen = -1;
- long _bclen = -1;
- double _nnz = -1;
- boolean _inmem = false;
-
- public VarStats( long rlen, long clen, long brlen, long bclen, long nnz, boolean inmem )
- {
- _rlen = rlen;
- _clen = clen;
- _brlen = brlen;
- _bclen = bclen;
- _nnz = nnz;
- _inmem = inmem;
- }
-
- /**
- *
- * @return
- */
- public double getSparsity()
- {
- return (_nnz<0) ? 1.0 : (double)_nnz/_rlen/_clen;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("VarStats: [");
- sb.append("rlen = ");
- sb.append(_rlen);
- sb.append(", clen = ");
- sb.append(_clen);
- sb.append(", nnz = ");
- sb.append(_nnz);
- sb.append(", inmem = ");
- sb.append(_inmem);
- sb.append("]");
-
- return sb.toString();
- }
-
- @Override
- public Object clone()
- {
- return new VarStats(_rlen, _clen, _brlen, _bclen,(long)_nnz, _inmem );
- }
-}
[09/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ProgramBlock.java
deleted file mode 100644
index b0cb2af..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ProgramBlock.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-
-public class ProgramBlock
-{
-
- protected static final Log LOG = LogFactory.getLog(ProgramBlock.class.getName());
- private static final boolean CHECK_MATRIX_SPARSITY = false;
-
- protected Program _prog; // pointer to Program this ProgramBlock is part of
- protected ArrayList<Instruction> _inst;
-
- //additional attributes for recompile
- protected StatementBlock _sb = null;
- protected long _tid = 0; //by default _t0
-
-
- public ProgramBlock(Program prog)
- throws DMLRuntimeException
- {
- _prog = prog;
- _inst = new ArrayList<Instruction>();
- }
-
-
- ////////////////////////////////////////////////
- // getters, setters and similar functionality
- ////////////////////////////////////////////////
-
- public Program getProgram(){
- return _prog;
- }
-
- public void setProgram(Program prog){
- _prog = prog;
- }
-
- public StatementBlock getStatementBlock(){
- return _sb;
- }
-
- public void setStatementBlock( StatementBlock sb ){
- _sb = sb;
- }
-
- public ArrayList<Instruction> getInstructions() {
- return _inst;
- }
-
- public Instruction getInstruction(int i) {
- return _inst.get(i);
- }
-
- public void setInstructions( ArrayList<Instruction> inst ) {
- _inst = inst;
- }
-
- public void addInstruction(Instruction inst) {
- _inst.add(inst);
- }
-
- public void addInstructions(ArrayList<Instruction> inst) {
- _inst.addAll(inst);
- }
-
- public int getNumInstructions() {
- return _inst.size();
- }
-
- public void setThreadID( long id ){
- _tid = id;
- }
-
-
- //////////////////////////////////////////////////////////
- // core instruction execution (program block, predicate)
- //////////////////////////////////////////////////////////
-
- /**
- * Executes this program block (incl recompilation if required).
- *
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> tmp = _inst;
-
- //dynamically recompile instructions if enabled and required
- try
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && _sb != null
- && _sb.requiresRecompilation() )
- {
- tmp = Recompiler.recompileHopsDag(_sb, _sb.get_hops(), ec.getVariables(), null, false, _tid);
-
- if( MLContextProxy.isActive() )
- tmp = MLContextProxy.performCleanupAfterRecompilation(tmp);
- }
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.incrementHOPRecompileTime(t1-t0);
- if( tmp!=_inst )
- Statistics.incrementHOPRecompileSB();
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to recompile program block.", ex);
- }
-
- //actual instruction execution
- executeInstructions(tmp, ec);
- }
-
- /**
- * Executes given predicate instructions (incl recompilation if required)
- *
- * @param inst
- * @param hops
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public ScalarObject executePredicate(ArrayList<Instruction> inst, Hop hops, boolean requiresRecompile, ValueType retType, ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ArrayList<Instruction> tmp = inst;
-
- //dynamically recompile instructions if enabled and required
- try {
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && requiresRecompile )
- {
- tmp = Recompiler.recompileHopsDag(hops, ec.getVariables(), null, false, _tid);
- }
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.incrementHOPRecompileTime(t1-t0);
- if( tmp!=inst )
- Statistics.incrementHOPRecompilePred();
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to recompile predicate instructions.", ex);
- }
-
- //actual instruction execution
- return executePredicateInstructions(tmp, retType, ec);
- }
-
- /**
- *
- * @param inst
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- protected void executeInstructions(ArrayList<Instruction> inst, ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- for (int i = 0; i < inst.size(); i++)
- {
- //indexed access required due to dynamic add
- Instruction currInst = inst.get(i);
-
- //execute instruction
- ec.updateDebugState(i);
- executeSingleInstruction(currInst, ec);
- }
- }
-
- /**
- *
- * @param inst
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- protected ScalarObject executePredicateInstructions(ArrayList<Instruction> inst, ValueType retType, ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ScalarObject ret = null;
- String retName = null;
-
- //execute all instructions
- for (int i = 0; i < inst.size(); i++)
- {
- //indexed access required due to debug mode
- Instruction currInst = inst.get(i);
- if( !isRemoveVariableInstruction(currInst) )
- {
- //execute instruction
- ec.updateDebugState(i);
- executeSingleInstruction(currInst, ec);
-
- //get last return name
- if(currInst instanceof ComputationCPInstruction )
- retName = ((ComputationCPInstruction) currInst).getOutputVariableName();
- else if(currInst instanceof VariableCPInstruction && ((VariableCPInstruction)currInst).getOutputVariableName()!=null)
- retName = ((VariableCPInstruction)currInst).getOutputVariableName();
- }
- }
-
- //get return value TODO: how do we differentiate literals and variables?
- ret = (ScalarObject) ec.getScalarInput(retName, retType, false);
-
- //execute rmvar instructions
- for (int i = 0; i < inst.size(); i++) {
- //indexed access required due to debug mode
- Instruction currInst = inst.get(i);
- if( isRemoveVariableInstruction(currInst) ) {
- ec.updateDebugState(i);
- executeSingleInstruction(currInst, ec);
- }
- }
-
- //check and correct scalar ret type (incl save double to int)
- if( ret.getValueType() != retType )
- switch( retType ) {
- case BOOLEAN: ret = new BooleanObject(ret.getName(),ret.getBooleanValue()); break;
- case INT: ret = new IntObject(ret.getName(),ret.getLongValue()); break;
- case DOUBLE: ret = new DoubleObject(ret.getName(),ret.getDoubleValue()); break;
- case STRING: ret = new StringObject(ret.getName(),ret.getStringValue()); break;
- default:
- //do nothing
- }
-
- return ret;
- }
-
- /**
- *
- *
- * @param currInst
- * @throws DMLRuntimeException
- */
- private void executeSingleInstruction( Instruction currInst, ExecutionContext ec )
- throws DMLRuntimeException
- {
- try
- {
- // start time measurement for statistics
- long t0 = (DMLScript.STATISTICS || LOG.isTraceEnabled()) ?
- System.nanoTime() : 0;
-
- // pre-process instruction (debug state, inst patching, listeners)
- Instruction tmp = currInst.preprocessInstruction( ec );
-
- // process actual instruction
- tmp.processInstruction( ec );
-
- // post-process instruction (debug)
- tmp.postprocessInstruction( ec );
-
- // maintain aggregate statistics
- if( DMLScript.STATISTICS) {
- Statistics.maintainCPHeavyHitters(
- tmp.getExtendedOpcode(), System.nanoTime()-t0);
- }
-
- // optional trace information (instruction and runtime)
- if( LOG.isTraceEnabled() ) {
- long t1 = System.nanoTime();
- String time = String.format("%.3f",((double)t1-t0)/1000000000);
- LOG.trace("Instruction: "+ tmp + " (executed in " + time + "s).");
- }
-
- // optional check for correct nnz and sparse/dense representation of all
- // variables in symbol table (for tracking source of wrong representation)
- if( CHECK_MATRIX_SPARSITY ) {
- checkSparsity( tmp, ec.getVariables() );
- }
- }
- catch (Exception e)
- {
- if (!DMLScript.ENABLE_DEBUG_MODE) {
- if ( e instanceof DMLScriptException)
- throw (DMLScriptException)e;
- else
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating instruction: " + currInst.toString() , e);
- }
- else {
- ec.handleDebugException(e);
- }
- }
- }
-
- /**
- *
- * @param inst
- * @return
- */
- private boolean isRemoveVariableInstruction(Instruction inst)
- {
- return ( inst instanceof VariableCPInstruction && ((VariableCPInstruction)inst).isRemoveVariable() );
- }
-
- public void printMe() {
- //System.out.println("***** INSTRUCTION BLOCK *****");
- for (Instruction i : this._inst) {
- i.printMe();
- }
- }
-
- /**
- *
- * @param lastInst
- * @param vars
- * @throws DMLRuntimeException
- */
- private void checkSparsity( Instruction lastInst, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- for( String varname : vars.keySet() )
- {
- Data dat = vars.get(varname);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject)dat;
- if( mo.isDirty() && !mo.isPartitioned() )
- {
- MatrixBlock mb = mo.acquireRead();
- boolean sparse1 = mb.isInSparseFormat();
- long nnz1 = mb.getNonZeros();
- synchronized( mb ) { //potential state change
- mb.recomputeNonZeros();
- mb.examSparsity();
- }
- boolean sparse2 = mb.isInSparseFormat();
- long nnz2 = mb.getNonZeros();
- mo.release();
-
- if( nnz1 != nnz2 )
- throw new DMLRuntimeException("Matrix nnz meta data was incorrect: ("+varname+", actual="+nnz1+", expected="+nnz2+", inst="+lastInst+")");
-
-
- if( sparse1 != sparse2 )
- throw new DMLRuntimeException("Matrix was in wrong data representation: ("+varname+", actual="+sparse1+", expected="+sparse2+", nnz="+nnz1+", inst="+lastInst+")");
- }
- }
- }
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for program blocks
- ///////////////////////////////////////////////////////////////////////////
-
- public int _beginLine, _beginColumn;
- public int _endLine, _endColumn;
-
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(int blp, int bcp, int elp, int ecp){
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in program block generated from statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/WhileProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/WhileProgramBlock.java
deleted file mode 100644
index cea071c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/WhileProgramBlock.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-
-public class WhileProgramBlock extends ProgramBlock
-{
-
- private ArrayList<Instruction> _predicate;
- private String _predicateResultVar;
- private ArrayList <Instruction> _exitInstructions ;
- private ArrayList<ProgramBlock> _childBlocks;
-
- public WhileProgramBlock(Program prog, ArrayList<Instruction> predicate) throws DMLRuntimeException{
- super(prog);
- _predicate = predicate;
- _predicateResultVar = findPredicateResultVar ();
- _exitInstructions = new ArrayList<Instruction>();
- _childBlocks = new ArrayList<ProgramBlock>();
- }
-
- public void printMe() {
-
- LOG.debug("***** while current block predicate inst: *****");
- for (Instruction cp : _predicate){
- cp.printMe();
- }
-
- for (ProgramBlock pb : this._childBlocks){
- pb.printMe();
- }
-
- LOG.debug("***** current block inst exit: *****");
- for (Instruction i : this._exitInstructions) {
- i.printMe();
- }
- }
-
-
-
-
- public void addProgramBlock(ProgramBlock childBlock) {
- _childBlocks.add(childBlock);
- }
-
- public void setExitInstructions2(ArrayList<Instruction> exitInstructions)
- { _exitInstructions = exitInstructions; }
-
- public void setExitInstructions1(ArrayList<Instruction> predicate)
- { _predicate = predicate; }
-
- public void addExitInstruction(Instruction inst)
- { _exitInstructions.add(inst); }
-
- public ArrayList<Instruction> getPredicate()
- { return _predicate; }
-
- public void setPredicate( ArrayList<Instruction> predicate )
- {
- _predicate = predicate;
-
- //update result var if non-empty predicate (otherwise,
- //do not overwrite varname predicate in predicateResultVar)
- if( _predicate != null && !_predicate.isEmpty() )
- _predicateResultVar = findPredicateResultVar();
- }
-
- public String getPredicateResultVar()
- { return _predicateResultVar; }
-
- public void setPredicateResultVar(String resultVar)
- { _predicateResultVar = resultVar; }
-
- public ArrayList<Instruction> getExitInstructions()
- { return _exitInstructions; }
-
- private BooleanObject executePredicate(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- BooleanObject result = null;
- try
- {
- if( _predicate!=null && !_predicate.isEmpty() )
- {
- if( _sb!=null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- WhileStatementBlock wsb = (WhileStatementBlock)_sb;
- Hop predicateOp = wsb.getPredicateHops();
- boolean recompile = wsb.requiresPredicateRecompilation();
- result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
- }
- else
- result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
- }
- else
- {
- //get result var
- ScalarObject scalarResult = null;
- Data resultData = ec.getVariable(_predicateResultVar);
- if ( resultData == null ) {
- //note: resultvar is a literal (can it be of any value type other than String, hence no literal/varname conflict)
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, true);
- }
- else {
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, false);
- }
-
- //check for invalid type String
- if (scalarResult instanceof StringObject)
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "\nWhile predicate variable "+ _predicateResultVar + " evaluated to string " + scalarResult + " which is not allowed for predicates in DML");
-
- //process result
- if( scalarResult instanceof BooleanObject )
- result = (BooleanObject)scalarResult;
- else
- result = new BooleanObject( scalarResult.getBooleanValue() ); //auto casting
- }
- }
- catch(Exception ex)
- {
- LOG.trace("\nWhile predicate variables: "+ ec.getVariables().toString());
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to evaluate the WHILE predicate.", ex);
- }
-
- //(guaranteed to be non-null, see executePredicate/getScalarInput)
- return result;
- }
-
- public void execute(ExecutionContext ec) throws DMLRuntimeException, DMLUnsupportedOperationException {
-
- BooleanObject predResult = executePredicate(ec);
-
- //execute while loop
- try
- {
- while(predResult.getBooleanValue())
- {
- //execute all child blocks
- for (int i=0 ; i < _childBlocks.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocks.get(i).execute(ec);
- }
-
- predResult = executePredicate(ec);
- }
- }
- catch(DMLScriptException e)
- {
- throw e;
- }
- catch(Exception e)
- {
- LOG.trace("\nWhile predicate variables: "+ ec.getVariables().toString());
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating while program block.", e);
- }
-
- //execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch(Exception e)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing while exit instructions.", e);
- }
- }
-
- public ArrayList<ProgramBlock> getChildBlocks() {
- return _childBlocks;
- }
-
- public void setChildBlocks(ArrayList<ProgramBlock> childs)
- {
- _childBlocks = childs;
- }
-
- private String findPredicateResultVar ( ) {
- String result = null;
- for ( Instruction si : _predicate ) {
- if ( si.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM && ((CPInstruction)si).getCPInstructionType() != CPINSTRUCTION_TYPE.Variable ) {
- result = ((ComputationCPInstruction) si).getOutputVariableName();
- }
- else if(si instanceof VariableCPInstruction && ((VariableCPInstruction)si).isVariableCastInstruction()){
- result = ((VariableCPInstruction)si).getOutputVariableName();
- }
- }
-
- return result;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in while program block generated from while statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/ByteBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/ByteBuffer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/ByteBuffer.java
deleted file mode 100644
index 97b6a18..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/ByteBuffer.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-/**
- * Wrapper for WriteBuffer byte array per matrix in order to
- * support matrix serialization outside global lock.
- *
- */
-public class ByteBuffer
-{
- private boolean _serialized;
- private boolean _sparse;
- private long _size;
-
- protected byte[] _bdata = null; //sparse matrix
- protected MatrixBlock _mdata = null; //dense matrix
-
- public ByteBuffer( long size )
- {
- _size = size;
- _serialized = false;
- }
-
- /**
- *
- * @param mb
- * @throws IOException
- */
- public void serializeMatrix( MatrixBlock mb )
- throws IOException
- {
- boolean sparseSrc = mb.isInSparseFormat(); //current representation
- boolean sparseTrgt = mb.evalSparseFormatOnDisk(); //intended target representation
- _sparse = sparseTrgt;
-
- try
- {
- if( _sparse ) //SPARSE/DENSE -> SPARSE
- {
- //deep serialize (for compression)
- if( CacheableData.CACHING_BUFFER_PAGECACHE )
- _bdata = PageCache.getPage((int)_size);
- if( _bdata==null )
- _bdata = new byte[(int)_size];
- DataOutput dout = new CacheDataOutput(_bdata);
- mb.write(dout);
- }
- else //SPARSE/DENSE -> DENSE
- {
- //change representation (if required), incl. free sparse
- //(in-memory representation, if dense on disk than if will
- //be guaranteed to be dense in memory as well)
- if( sparseSrc )
- mb.examSparsity();
-
- //shallow serialize
- _mdata = mb;
- }
- }
- catch(Exception ex)
- {
- throw new IOException("Failed to serialize matrix block.", ex);
- }
-
- _serialized = true;
- }
-
- /**
- *
- * @return
- * @throws IOException
- */
- public MatrixBlock deserializeMatrix()
- throws IOException
- {
- MatrixBlock ret = null;
-
- if( _sparse )
- {
- //ByteArrayInputStream bis = new ByteArrayInputStream(_bdata);
- //DataInputStream din = new DataInputStream(bis);
- CacheDataInput din = new CacheDataInput(_bdata);
- ret = new MatrixBlock();
- ret.readFields(din);
- }
- else
- {
- ret = _mdata;
- }
-
- return ret;
- }
-
- /**
- *
- * @param fname
- * @throws IOException
- */
- public void evictBuffer( String fname )
- throws IOException
- {
- if( _sparse )
- {
- //write out byte serialized array
- LocalFileUtils.writeByteArrayToLocal(fname, _bdata);
- }
- else
- {
- //serialize matrix to output stream
- LocalFileUtils.writeMatrixBlockToLocal(fname, _mdata);
- }
- }
-
- /**
- * Returns the buffer size in bytes.
- *
- * @return
- */
- public long getSize()
- {
- return _size;
- }
-
- /**
- *
- * @return
- */
- public boolean isInSparseFormat()
- {
- return _sparse;
- }
-
- public void freeMemory()
- {
- //clear strong references to buffer/matrix
- if( _sparse )
- {
- if( CacheableData.CACHING_BUFFER_PAGECACHE )
- PageCache.putPage(_bdata);
- _bdata = null;
- }
- else
- {
- _mdata = null;
- }
- }
-
- /**
- *
- */
- public void checkSerialized()
- {
- if( _serialized )
- return;
-
- while( !_serialized )
- {
- try{Thread.sleep(1);} catch(Exception e) {}
- }
- }
-
- /**
- * Determines if byte buffer can hold the given size given this specific matrix block.
- * This call is consistent with 'serializeMatrix' and allows for internal optimization
- * according to dense/sparse representation.
- *
- * @param size
- * @param mb
- * @return
- */
- public static boolean isValidCapacity( long size, MatrixBlock mb )
- {
- boolean sparseTrgt = mb.evalSparseFormatOnDisk(); //intended target representation
-
- if( sparseTrgt ) //SPARSE
- {
- // since sparse matrix blocks are serialized into a byte representation
- // the buffer buffer can hold at most 2GB in size
- return ( size <= Integer.MAX_VALUE );
- }
- else //DENSE
- {
- // since for dense matrix blocks we use a shallow serialize (strong reference),
- // the byte buffer can hold any size (currently upper bounded by 16GB)
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataInput.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataInput.java
deleted file mode 100644
index f2802a3..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataInput.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.DataInput;
-import java.io.IOException;
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataInput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-
-public class CacheDataInput implements DataInput, MatrixBlockDataInput
-{
-
- protected byte[] _buff;
- protected int _bufflen;
- protected int _count;
-
- public CacheDataInput( byte[] mem )
- {
- _buff = mem;
- _bufflen = _buff.length;
- _count = 0;
- }
-
- @Override
- public void readFully(byte[] b) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void readFully(byte[] b, int off, int len) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public int skipBytes(int n) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public boolean readBoolean()
- throws IOException
- {
- //mask to adhere to the input stream semantic
- return ( (_buff[_count++] & 0xFF) != 0 );
- }
-
- @Override
- public byte readByte()
- throws IOException
- {
- //mask to adhere to the input stream semantic
- return (byte) (_buff[_count++] & 0xFF);
- }
-
- @Override
- public int readUnsignedByte() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public short readShort() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public int readUnsignedShort() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public char readChar() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public int readInt()
- throws IOException
- {
- int ret = baToInt(_buff, _count);
- _count += 4;
-
- return ret;
- }
-
- @Override
- public long readLong()
- throws IOException
- {
- long ret = baToLong(_buff, _count);
- _count += 8;
-
- return ret;
- }
-
- @Override
- public float readFloat() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public double readDouble()
- throws IOException
- {
- long tmp = baToLong(_buff, _count);
- double tmp2 = Double.longBitsToDouble(tmp);
- _count += 8;
-
- return tmp2;
- }
-
- @Override
- public String readLine() throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public String readUTF() throws IOException {
- throw new IOException("Not supported.");
- }
-
- ///////////////////////////////////////////////
- // Implementation of MatrixBlockDSMDataOutput
- ///////////////////////////////////////////////
-
- @Override
- public long readDoubleArray(int len, double[] varr)
- throws IOException
- {
- //counter for non-zero elements
- long nnz = 0;
-
- int off = _count;
- for( int i=0; i<len; i++ )
- {
- //core deserialization
- long tmp = baToLong(_buff, off+i*8);
- varr[i] = Double.longBitsToDouble( tmp );
-
- //nnz maintenance
- nnz += (varr[i]!=0) ? 1 : 0;
- }
- _count = off + len*8;
-
- return nnz;
- }
-
- @Override
- public long readSparseRows(int rlen, SparseRow[] rows)
- throws IOException
- {
- //counter for non-zero elements
- long nnz = 0;
-
- //read all individual sparse rows from input
- for( int i=0; i<rlen; i++ )
- {
- int lnnz = readInt();
-
- if( lnnz > 0 ) //non-zero row
- {
- //get handle to sparse (allocate if necessary)
- if( rows[i] == null )
- rows[i] = new SparseRow(lnnz);
- SparseRow arow = rows[i];
-
- //read single sparse row
- for( int j=0; j<lnnz; j++ )
- {
- int aix = baToInt(_buff, _count);
- long tmp = baToLong(_buff, _count+4);
- double aval = Double.longBitsToDouble( tmp );
- arow.append(aix, aval);
- _count+=12;
- }
-
- nnz += lnnz;
- }
- }
-
- return nnz;
- }
-
- /**
- *
- * @param a
- * @param off
- * @return
- */
- private static int baToInt( byte[] ba, final int off )
- {
- //shift and add 4 bytes into single int
- return ((ba[off+0] & 0xFF) << 24) +
- ((ba[off+1] & 0xFF) << 16) +
- ((ba[off+2] & 0xFF) << 8) +
- ((ba[off+3] & 0xFF) << 0);
- }
-
- /**
- *
- * @param a
- * @param off
- * @return
- */
- private static long baToLong( byte[] ba, final int off )
- {
- //shift and add 8 bytes into single long
- return ((long)(ba[off+0] & 0xFF) << 56) +
- ((long)(ba[off+1] & 0xFF) << 48) +
- ((long)(ba[off+2] & 0xFF) << 40) +
- ((long)(ba[off+3] & 0xFF) << 32) +
- ((long)(ba[off+4] & 0xFF) << 24) +
- ((long)(ba[off+5] & 0xFF) << 16) +
- ((long)(ba[off+6] & 0xFF) << 8) +
- ((long)(ba[off+7] & 0xFF) << 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataOutput.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataOutput.java
deleted file mode 100644
index d821319..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheDataOutput.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlockDataOutput;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-
-/**
- * Customer DataOutput to serialize directly into the given byte array.
- *
- *
- */
-public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
-{
-
- protected byte[] _buff;
- protected int _bufflen;
- protected int _count;
-
- public CacheDataOutput( byte[] mem )
- {
- _buff = mem;
- _bufflen = _buff.length;
- _count = 0;
- }
-
- @Override
- public void write(int b)
- throws IOException
- {
- _buff[_count++] = (byte)b;
- }
-
- @Override
- public void write(byte[] b)
- throws IOException
- {
- System.arraycopy(b, 0, _buff, _count, b.length);
- _count += b.length;
- }
-
- @Override
- public void write(byte[] b, int off, int len)
- throws IOException
- {
- System.arraycopy(b, off, _buff, _count, len);
- _count += len;
- }
-
- @Override
- public void writeBoolean(boolean v)
- throws IOException
- {
- _buff[_count++] = (byte)( v ? 1 : 0 );
- }
-
-
- @Override
- public void writeInt(int v)
- throws IOException
- {
- intToBa(v, _buff, _count);
- _count += 4;
- }
-
- @Override
- public void writeDouble(double v)
- throws IOException
- {
- long tmp = Double.doubleToRawLongBits(v);
- longToBa(tmp, _buff, _count);
- _count += 8;
- }
-
- @Override
- public void writeByte(int v) throws IOException {
- _buff[_count++] = (byte) v;
- }
-
- @Override
- public void writeBytes(String s) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void writeChar(int v) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void writeChars(String s) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void writeFloat(float v) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void writeLong(long v) throws IOException {
- longToBa(v, _buff, _count);
- _count += 8;
- }
-
- @Override
- public void writeShort(int v) throws IOException {
- throw new IOException("Not supported.");
- }
-
- @Override
- public void writeUTF(String s) throws IOException {
- throw new IOException("Not supported.");
- }
-
-
- ///////////////////////////////////////////////
- // Implementation of MatrixBlockDSMDataOutput
- ///////////////////////////////////////////////
-
- @Override
- public void writeDoubleArray(int len, double[] varr)
- throws IOException
- {
- //original buffer offset
- int off = _count;
-
- //serialize entire array into buffer
- for( int i=0; i<len; i++ )
- {
- long tmp = Double.doubleToRawLongBits(varr[i]);
- longToBa(tmp, _buff, off+i*8);
- }
-
- //update buffer offset
- _count = off + len*8;
- }
-
- @Override
- public void writeSparseRows(int rlen, SparseRow[] rows)
- throws IOException
- {
- int lrlen = Math.min(rows.length, rlen);
-
- //process existing rows
- for( int i=0; i<lrlen; i++ )
- {
- SparseRow arow = rows[i];
- if( arow!=null && !arow.isEmpty() )
- {
- int alen = arow.size();
- int[] aix = arow.getIndexContainer();
- double[] avals = arow.getValueContainer();
-
- writeInt( alen );
-
- for( int j=0; j<alen; j++ )
- {
- intToBa(aix[j], _buff, _count);
- long tmp2 = Double.doubleToRawLongBits(avals[j]);
- longToBa(tmp2, _buff, _count+4);
- _count += 12;
- }
- }
- else
- writeInt( 0 );
- }
-
- //process remaining empty rows
- for( int i=lrlen; i<rlen; i++ )
- writeInt( 0 );
- }
-
- /**
- *
- * @param val
- * @param ba
- * @param off
- */
- private static void intToBa( final int val, byte[] ba, final int off )
- {
- //shift and mask out 4 bytes
- ba[ off+0 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 0) & 0xFF);
- }
-
- /**
- *
- * @param val
- * @param ba
- * @param off
- */
- private static void longToBa( final long val, byte[] ba, final int off )
- {
- //shift and mask out 8 bytes
- ba[ off+0 ] = (byte)((val >>> 56) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 48) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 40) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 32) & 0xFF);
- ba[ off+4 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+5 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+6 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+7 ] = (byte)((val >>> 0) & 0xFF);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheException.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheException.java
deleted file mode 100644
index 3516b8b..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheException.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-public class CacheException extends DMLRuntimeException
-{
-
-
- private static final long serialVersionUID = 1L;
-
- public CacheException ()
- {
- super ("Cache Exception");
- }
-
- public CacheException (String message)
- {
- super (message);
- }
-
- public CacheException (Exception cause)
- {
- super (cause);
- }
-
- public CacheException (String message, Exception cause)
- {
- super (message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheIOException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheIOException.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheIOException.java
deleted file mode 100644
index 6efda8d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheIOException.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-
-public class CacheIOException extends CacheException
-{
-
-
- private static final long serialVersionUID = 1L;
-
- public CacheIOException ()
- {
- super ();
- }
-
- public CacheIOException (String message)
- {
- super (message);
- }
-
- public CacheIOException (Exception cause)
- {
- super (cause);
- }
-
- public CacheIOException (String message, Exception cause)
- {
- super (message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatistics.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatistics.java
deleted file mode 100644
index e12b586..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatistics.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This singleton provides basic caching statistics in CP.
- *
- * 1) Hit statistics for caching (mem, fs, hdfs, total)
- *
- * NOTE: In order to provide accurate statistics in multi-threaded
- * synchronized increments are required. Since those functions are
- * called potentially very often, we use atomic increments
- * (compare and swap) instead of heavy-weight 'synchronized' methods.
- *
- */
-public class CacheStatistics
-{
-
- //enum used for MR counters
- public enum Stat {
- CACHE_HITS_MEM,
- CACHE_HITS_FSBUFF,
- CACHE_HITS_FS,
- CACHE_HITS_HDFS,
- CACHE_WRITES_FSBUFF,
- CACHE_WRITES_FS,
- CACHE_WRITES_HDFS,
- CACHE_TIME_ACQR, //acquire read
- CACHE_TIME_ACQM, //acquire read
- CACHE_TIME_RLS, //release
- CACHE_TIME_EXP, //export
- }
-
- //hit statistics (for acquire read)
- private static AtomicLong _numHitsTotal = null;
- private static AtomicLong _numHitsMem = null;
- private static AtomicLong _numHitsFSBuff = null;
- private static AtomicLong _numHitsFS = null;
- private static AtomicLong _numHitsHDFS = null;
-
- //write statistics caching
- private static AtomicLong _numWritesFSBuff = null;
- private static AtomicLong _numWritesFS = null;
- private static AtomicLong _numWritesHDFS = null;
-
- //time statistics caching
- private static AtomicLong _ctimeAcquireR = null; //in nano sec
- private static AtomicLong _ctimeAcquireM = null; //in nano sec
- private static AtomicLong _ctimeRelease = null; //in nano sec
- private static AtomicLong _ctimeExport = null; //in nano sec
-
- static
- {
- reset();
- }
-
- public static void reset()
- {
- _numHitsTotal = new AtomicLong(0);
- _numHitsMem = new AtomicLong(0);
- _numHitsFSBuff = new AtomicLong(0);
- _numHitsFS = new AtomicLong(0);
- _numHitsHDFS = new AtomicLong(0);
-
- _numWritesFSBuff = new AtomicLong(0);
- _numWritesFS = new AtomicLong(0);
- _numWritesHDFS = new AtomicLong(0);
-
- _ctimeAcquireR = new AtomicLong(0);
- _ctimeAcquireM = new AtomicLong(0);
- _ctimeRelease = new AtomicLong(0);
- _ctimeExport = new AtomicLong(0);
- }
-
- public static void incrementTotalHits()
- {
- _numHitsTotal.incrementAndGet();
- }
-
- public static void incrementTotalHits(int delta)
- {
- _numHitsTotal.addAndGet(delta);
- }
-
- public static long getTotalHits()
- {
- return _numHitsTotal.get();
- }
-
- public static void incrementMemHits()
- {
- _numHitsMem.incrementAndGet();
- }
-
- public static void incrementMemHits(int delta)
- {
- _numHitsMem.addAndGet(delta);
- }
-
- public static long getMemHits()
- {
- return _numHitsMem.get();
- }
-
- public static void incrementFSBuffHits()
- {
- _numHitsFSBuff.incrementAndGet();
- }
-
- public static void incrementFSBuffHits( int delta )
- {
- _numHitsFSBuff.addAndGet(delta);
- }
-
- public static long getFSBuffHits()
- {
- return _numHitsFSBuff.get();
- }
-
- public static void incrementFSHits()
- {
- _numHitsFS.incrementAndGet();
- }
-
- public static void incrementFSHits(int delta)
- {
- _numHitsFS.addAndGet(delta);
- }
-
- public static long getFSHits()
- {
- return _numHitsFS.get();
- }
-
- public static void incrementHDFSHits()
- {
- _numHitsHDFS.incrementAndGet();
- }
-
- public static void incrementHDFSHits(int delta)
- {
- _numHitsHDFS.addAndGet(delta);
- }
-
- public static long getHDFSHits()
- {
- return _numHitsHDFS.get();
- }
-
- public static void incrementFSBuffWrites()
- {
- _numWritesFSBuff.incrementAndGet();
- }
-
- public static void incrementFSBuffWrites(int delta)
- {
- _numWritesFSBuff.addAndGet(delta);
- }
-
- public static long getFSBuffWrites()
- {
- return _numWritesFSBuff.get();
- }
-
- public static void incrementFSWrites()
- {
- _numWritesFS.incrementAndGet();
- }
-
- public static void incrementFSWrites(int delta)
- {
- _numWritesFS.addAndGet(delta);
- }
-
- public static long getFSWrites()
- {
- return _numWritesFS.get();
- }
-
- public static void incrementHDFSWrites()
- {
- _numWritesHDFS.incrementAndGet();
- }
-
- public static void incrementHDFSWrites(int delta)
- {
- _numWritesHDFS.addAndGet(delta);
- }
-
- public static long getHDFSWrites()
- {
- return _numWritesHDFS.get();
- }
-
- public static void incrementAcquireRTime(long delta)
- {
- _ctimeAcquireR.addAndGet(delta);
- }
-
- public static long getAcquireRTime()
- {
- return _ctimeAcquireR.get();
- }
-
- public static void incrementAcquireMTime(long delta)
- {
- _ctimeAcquireM.addAndGet(delta);
- }
-
- public static long getAcquireMTime()
- {
- return _ctimeAcquireM.get();
- }
-
- public static void incrementReleaseTime(long delta)
- {
- _ctimeRelease.addAndGet(delta);
- }
-
- public static long getReleaseTime()
- {
- return _ctimeRelease.get();
- }
-
-
- public static void incrementExportTime(long delta)
- {
- _ctimeExport.addAndGet(delta);
- }
-
- public static long getExportTime()
- {
- return _ctimeExport.get();
- }
-
-
- public static String displayHits()
- {
- StringBuilder sb = new StringBuilder();
- sb.append(_numHitsMem.get());
- sb.append("/");
- sb.append(_numHitsFSBuff.get());
- sb.append("/");
- sb.append(_numHitsFS.get());
- sb.append("/");
- sb.append(_numHitsHDFS.get());
-
-
- return sb.toString();
- }
-
- public static String displayWrites()
- {
- StringBuilder sb = new StringBuilder();
- sb.append(_numWritesFSBuff.get());
- sb.append("/");
- sb.append(_numWritesFS.get());
- sb.append("/");
- sb.append(_numWritesHDFS.get());
-
-
- return sb.toString();
- }
-
- public static String displayTime()
- {
- StringBuilder sb = new StringBuilder();
- sb.append(String.format("%.3f", ((double)_ctimeAcquireR.get())/1000000000)); //in sec
- sb.append("/");
- sb.append(String.format("%.3f", ((double)_ctimeAcquireM.get())/1000000000)); //in sec
- sb.append("/");
- sb.append(String.format("%.3f", ((double)_ctimeRelease.get())/1000000000)); //in sec
- sb.append("/");
- sb.append(String.format("%.3f", ((double)_ctimeExport.get())/1000000000)); //in sec
-
- ;
-
- return sb.toString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatusException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatusException.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatusException.java
deleted file mode 100644
index 1cefff7..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheStatusException.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-
-public class CacheStatusException extends CacheException
-{
-
-
- private static final long serialVersionUID = 1L;
-
- public CacheStatusException (String message)
- {
- super (message);
- }
-
- public CacheStatusException (Exception cause)
- {
- super (cause);
- }
-
- public CacheStatusException (String message, Exception cause)
- {
- super (message, cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheableData.java
deleted file mode 100644
index 2785e64..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/CacheableData.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.LazyWriteBuffer.RPolicy;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-
-/**
- * Each object of this class is a cache envelope for some large piece of data
- * called "data blob". (I prefer "blob" to "block" to avoid ambiguity.) For
- * example, the body of a matrix can be the data blob. The term "data blob"
- * refers strictly to the cacheable portion of the data object, often excluding
- * metadata and auxiliary parameters, as defined in the subclasses.
- * Under the protection of the envelope, the data blob may be evicted to
- * the file system; then the subclass must set its reference to <code>null</code>
- * to allow Java garbage collection. If other parts of the system continue
- * keep references to the data blob, its eviction will not release any memory.
- * To make the eviction meaningful, the rest of the system
- * must dispose of all references prior to giving the permission for eviction.
- *
- */
-public abstract class CacheableData extends Data
-{
-
- private static final long serialVersionUID = -413810592207212835L;
-
- protected static final Log LOG = LogFactory.getLog(CacheableData.class.getName());
-
- public static final long CACHING_THRESHOLD = 4*1024; //obj not s.t. caching if below threshold [in bytes]
- public static final double CACHING_BUFFER_SIZE = 0.15;
- public static final RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
- public static final boolean CACHING_BUFFER_PAGECACHE = false;
- public static final boolean CACHING_WRITE_CACHE_ON_READ = false;
-
- public static final String CACHING_COUNTER_GROUP_NAME = "SystemML Caching Counters";
-
-
- //flag indicating if caching is turned on (eviction writes only happen if activeFlag is true)
- private static boolean _activeFlag = false;
-
- public static String cacheEvictionLocalFilePath = null; //set during init
- public static String cacheEvictionLocalFilePrefix = "cache";
- public static final String cacheEvictionLocalFileExtension = ".dat";
-
- /**
- * Defines all possible cache status types for a data blob.
- * An object of class {@link CacheableData} can be in one of the following
- * five status types:
- *
- * <code>EMPTY</code>: Either there is no data blob at all, or the data blob
- * resides in a specified import file and has never been downloaded yet.
- * <code>READ</code>: The data blob is in main memory; one or more threads are
- * referencing and reading it (shared "read-only" lock). This status uses a
- * counter. Eviction is NOT allowed.
- * <code>MODIFY</code>: The data blob is in main memory; exactly one thread is
- * referencing and modifying it (exclusive "write" lock). Eviction is NOT allowed.
- * <code>CACHED</code>: The data blob is in main memory, and nobody is using nor referencing it.
- * There is always an persistent recovery object for it
- **/
- protected enum CacheStatus {
- EMPTY,
- READ,
- MODIFY,
- CACHED,
- CACHED_NOWRITE,
- };
-
- private static IDSequence _seq = null;
-
- static
- {
- _seq = new IDSequence();
- }
-
- /**
- * The unique (JVM-wide) ID of a cacheable data object; to ensure unique IDs across JVMs, we
- * concatenate filenames with a unique prefix (map task ID).
- */
- private final int _uniqueID;
-
- /**
- * The cache status of the data blob (whether it can be or is evicted, etc.)
- */
- private CacheStatus _cacheStatus = null;
- private int _numReadThreads = 0;
-
- protected CacheableData (DataType dt, ValueType vt)
- {
- super (dt, vt);
-
- _uniqueID = (int)_seq.getNextID();
-
- _cacheStatus = CacheStatus.EMPTY;
- _numReadThreads = 0;
- }
-
- // --------- ABSTRACT LOW-LEVEL CACHE I/O OPERATIONS ----------
-
- /**
- * Checks if the data blob reference points to some in-memory object.
- * This method is called when releasing the (last) lock. Do not call
- * this method for a blob that has been evicted.
- *
- * @return <code>true</code> if the blob is in main memory and the
- * reference points to it;
- * <code>false</code> if the blob reference is <code>null</code>.
- */
- protected abstract boolean isBlobPresent();
-
- /**
- * Low-level cache I/O method that physically evicts the data blob from
- * main memory. Must be defined by a subclass, never called by users.
- * @param mb
- *
- * @throws CacheIOException if the eviction fails, the data blob
- * remains as it was at the start.
- */
- protected abstract void evictBlobFromMemory(MatrixBlock mb)
- throws CacheIOException;
-
- /**
- * Low-level cache I/O method that physically restores the data blob to
- * main memory. Must be defined by a subclass, never called by users.
- *
- * @throws CacheIOException if the restore fails, the data blob
- * remains as it was at the start.
- * @throws CacheAssignmentException if the restored blob cannot be assigned
- * to this envelope.
- */
- protected abstract void restoreBlobIntoMemory()
- throws CacheIOException;
-
- /**
- * Low-level cache I/O method that deletes the file containing the
- * evicted data blob, without reading it.
- * Must be defined by a subclass, never called by users.
- */
- protected abstract void freeEvictedBlob();
-
- /**
- *
- */
- protected abstract boolean isBelowCachingThreshold();
-
-
- // ------------- IMPLEMENTED CACHE LOGIC METHODS --------------
-
- protected int getUniqueCacheID()
- {
- return _uniqueID;
- }
-
- /**
- * This method "acquires the lock" to ensure that the data blob is in main memory
- * (not evicted) while it is being accessed. When called, the method will try to
- * restore the blob if it has been evicted. There are two kinds of locks it may
- * acquire: a shared "read" lock (if the argument is <code>false</code>) or the
- * exclusive "modify" lock (if the argument is <code>true</code>).
- * The method can fail in three ways:
- * (1) if there is lock status conflict;
- * (2) if there is not enough cache memory to restore the blob;
- * (3) if the restore method returns an error.
- * The method locks the data blob in memory (which disables eviction) and updates
- * its last-access timestamp. For the shared "read" lock, acquiring a new lock
- * increments the associated count. The "read" count has to be decremented once
- * the blob is no longer used, which may re-enable eviction. This method has to
- * be called only once per matrix operation and coupled with {@link #release()},
- * because it increments the lock count and the other method decrements this count.
- *
- * @param isModify : <code>true</code> for the exclusive "modify" lock,
- * <code>false</code> for a shared "read" lock.
- * @throws CacheException
- */
- protected void acquire (boolean isModify, boolean restore)
- throws CacheException
- {
- switch ( _cacheStatus )
- {
- case CACHED:
- if(restore)
- restoreBlobIntoMemory();
- case CACHED_NOWRITE:
- case EMPTY:
- if (isModify)
- setModify();
- else
- addOneRead();
- break;
- case READ:
- if (isModify)
- throw new CacheStatusException ("READ-MODIFY not allowed.");
- else
- addOneRead();
- break;
- case MODIFY:
- throw new CacheStatusException ("MODIFY-MODIFY not allowed.");
- }
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Acquired lock on " + this.getDebugName() + ", status: " + this.getStatusAsString() );
- }
-
-
- /**
- * Call this method to permit eviction for the stored data blob, or to
- * decrement its "read" count if it is "read"-locked by other threads.
- * It is expected that you eliminate all external references to the blob
- * prior to calling this method, because otherwise eviction will
- * duplicate the blob, but not release memory. This method has to be
- * called only once per process and coupled with {@link #acquire(boolean)},
- * because it decrements the lock count and the other method increments
- * the lock count.
- *
- * @throws CacheException
- */
- protected void release(boolean cacheNoWrite)
- throws CacheException
- {
- switch ( _cacheStatus )
- {
- case EMPTY:
- case CACHED:
- case CACHED_NOWRITE:
- throw new CacheStatusException("Redundant release.");
- case READ:
- removeOneRead( isBlobPresent(), cacheNoWrite );
- break;
- case MODIFY:
- if ( isBlobPresent() )
- setCached();
- else
- setEmpty();
- break;
- }
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Released lock on " + this.getDebugName() + ", status: " + this.getStatusAsString());
-
- }
-
-
- // **************************************************
- // *** ***
- // *** CACHE STATUS FIELD - CLASSES AND METHODS ***
- // *** ***
- // **************************************************
-
-
- public String getStatusAsString()
- {
- return _cacheStatus.toString();
- }
-
- //TODO isCached is only public for access from SparkExectionContext, once we can assume
- //the existence of spark libraries, we can move the related code to MatrixObject and
- //make this method protected again
- public boolean isCached(boolean inclCachedNoWrite)
- {
- if( inclCachedNoWrite )
- return (_cacheStatus == CacheStatus.CACHED || _cacheStatus == CacheStatus.CACHED_NOWRITE);
- else
- return (_cacheStatus == CacheStatus.CACHED);
- }
-
- protected boolean isEmpty(boolean inclCachedNoWrite)
- {
- if( inclCachedNoWrite )
- return (_cacheStatus == CacheStatus.EMPTY || _cacheStatus == CacheStatus.CACHED_NOWRITE);
- else
- return (_cacheStatus == CacheStatus.EMPTY);
- }
-
- protected boolean isModify()
- {
- return (_cacheStatus == CacheStatus.MODIFY);
- }
-
- protected void setEmpty()
- {
- _cacheStatus = CacheStatus.EMPTY;
- }
-
- protected void setModify()
- {
- _cacheStatus = CacheStatus.MODIFY;
- }
-
- protected void setCached()
- {
- _cacheStatus = CacheStatus.CACHED;
- }
-
- protected void addOneRead()
- {
- _numReadThreads ++;
- _cacheStatus = CacheStatus.READ;
- }
-
- protected void removeOneRead(boolean doesBlobExist, boolean cacheNoWrite)
- {
- _numReadThreads --;
- if (_numReadThreads == 0) {
- if( cacheNoWrite )
- _cacheStatus = (doesBlobExist ?
- CacheStatus.CACHED_NOWRITE : CacheStatus.EMPTY);
- else
- _cacheStatus = (doesBlobExist ?
- CacheStatus.CACHED : CacheStatus.EMPTY);
- }
- }
-
- protected boolean isAvailableToRead()
- {
- return ( _cacheStatus == CacheStatus.EMPTY
- || _cacheStatus == CacheStatus.CACHED
- || _cacheStatus == CacheStatus.CACHED_NOWRITE
- || _cacheStatus == CacheStatus.READ);
- }
-
- protected boolean isAvailableToModify()
- {
- return ( _cacheStatus == CacheStatus.EMPTY
- || _cacheStatus == CacheStatus.CACHED
- || _cacheStatus == CacheStatus.CACHED_NOWRITE);
- }
-
- // --------- STATIC CACHE INIT/CLEANUP OPERATIONS ----------
-
-
- /**
- *
- */
- public synchronized static void cleanupCacheDir()
- {
- //cleanup remaining cached writes
- LazyWriteBuffer.cleanup();
-
- //delete cache dir and files
- cleanupCacheDir(true);
- }
-
- /**
- * Deletes the DML-script-specific caching working dir.
- *
- * @param withDir
- */
- public synchronized static void cleanupCacheDir(boolean withDir)
- {
- //get directory name
- String dir = cacheEvictionLocalFilePath;
-
- //clean files with cache prefix
- if( dir != null ) //if previous init cache
- {
- File fdir = new File(dir);
- if( fdir.exists()){ //just for robustness
- File[] files = fdir.listFiles();
- for( File f : files )
- if( f.getName().startsWith(cacheEvictionLocalFilePrefix) )
- f.delete();
- if( withDir )
- fdir.delete(); //deletes dir only if empty
- }
- }
-
- _activeFlag = false;
- }
-
- /**
- * Inits caching with the default uuid of DMLScript
- * @throws IOException
- */
- public synchronized static void initCaching()
- throws IOException
- {
- initCaching(DMLScript.getUUID());
- }
-
- /**
- * Creates the DML-script-specific caching working dir.
- *
- * Takes the UUID in order to allow for custom uuid, e.g., for remote parfor caching
- *
- * @throws IOException
- */
- public synchronized static void initCaching( String uuid )
- throws IOException
- {
- try
- {
- String dir = LocalFileUtils.getWorkingDir( LocalFileUtils.CATEGORY_CACHE );
- LocalFileUtils.createLocalFileIfNotExist(dir);
- cacheEvictionLocalFilePath = dir;
- }
- catch(DMLRuntimeException e)
- {
- throw new IOException(e);
- }
-
- //init write-ahead buffer
- LazyWriteBuffer.init();
-
- _activeFlag = true; //turn on caching
- }
-
- public static synchronized boolean isCachingActive()
- {
- return _activeFlag;
- }
-
- public static synchronized void disableCaching()
- {
- _activeFlag = false;
- }
-
- public static synchronized void enableCaching()
- {
- _activeFlag = true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/LazyWriteBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/LazyWriteBuffer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/LazyWriteBuffer.java
deleted file mode 100644
index 3d60788..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/LazyWriteBuffer.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-/**
- *
- *
- */
-public class LazyWriteBuffer
-{
-
- public enum RPolicy{
- FIFO,
- LRU
- }
-
- //global size limit in bytes
- private static long _limit;
-
- //current size in bytes
- private static long _size;
-
- //eviction queue of <filename,buffer> pairs (implemented via linked hash map
- //for (1) queue semantics and (2) constant time get/insert/delete operations)
- private static EvictionQueue _mQueue;
-
- static
- {
- //obtain the logical buffer size in bytes
- long maxMem = InfrastructureAnalyzer.getLocalMaxMemory();
- _limit = (long)(CacheableData.CACHING_BUFFER_SIZE * maxMem);
- }
-
- /**
- *
- * @param fname
- * @param mb
- * @throws IOException
- */
- public static void writeMatrix( String fname, MatrixBlock mb )
- throws IOException
- {
- long lSize = mb.getExactSizeOnDisk();
- boolean requiresWrite = ( lSize > _limit //global buffer limit
- || !ByteBuffer.isValidCapacity(lSize, mb) ); //local buffer limit
-
- if( !requiresWrite ) //if it fits in writebuffer
- {
- ByteBuffer bbuff = null;
-
- //modify buffer pool
- synchronized( _mQueue )
- {
- //evict matrices to make room (by default FIFO)
- while( _size+lSize >= _limit )
- {
- //remove first entry from eviction queue
- Entry<String, ByteBuffer> entry = _mQueue.removeFirst();
- String ftmp = entry.getKey();
- ByteBuffer tmp = entry.getValue();
-
- if( tmp != null )
- {
- //wait for pending serialization
- tmp.checkSerialized();
-
- //evict matrix
- tmp.evictBuffer(ftmp);
- tmp.freeMemory();
- _size-=tmp.getSize();
-
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementFSWrites();
- }
- }
-
- //create buffer (reserve mem), and lock
- bbuff = new ByteBuffer( lSize );
-
- //put placeholder into buffer pool
- _mQueue.addLast(fname, bbuff);
- _size += lSize;
- }
-
- //serialize matrix (outside synchronized critical path)
- bbuff.serializeMatrix(mb);
-
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementFSBuffWrites();
- }
- else
- {
- //write directly to local FS (bypass buffer if too large)
- LocalFileUtils.writeMatrixBlockToLocal(fname, mb);
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementFSWrites();
- }
- }
-
- /**
- *
- * @param fname
- */
- public static void deleteMatrix( String fname )
- {
- boolean requiresDelete = true;
-
- synchronized( _mQueue )
- {
- //remove queue entry
- ByteBuffer ldata = _mQueue.remove(fname);
- if( ldata != null )
- {
- _size -= ldata.getSize();
- requiresDelete = false;
- ldata.freeMemory(); //cleanup
- }
- }
-
- //delete from FS if required
- if( requiresDelete )
- LocalFileUtils.deleteFileIfExists(fname, true);
- }
-
- /**
- *
- * @param fname
- * @return
- * @throws IOException
- */
- public static MatrixBlock readMatrix( String fname )
- throws IOException
- {
- MatrixBlock mb = null;
- ByteBuffer ldata = null;
-
- //probe write buffer
- synchronized( _mQueue )
- {
- ldata = _mQueue.get(fname);
-
- //modify eviction order (accordingly to access)
- if( CacheableData.CACHING_BUFFER_POLICY == RPolicy.LRU
- && ldata != null )
- {
- //reinsert entry at end of eviction queue
- _mQueue.remove( fname );
- _mQueue.addLast( fname, ldata );
- }
- }
-
- //deserialize or read from FS if required
- if( ldata != null )
- {
- mb = ldata.deserializeMatrix();
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementFSBuffHits();
- }
- else
- {
- mb = LocalFileUtils.readMatrixBlockFromLocal(fname); //read from FS
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementFSHits();
- }
-
- return mb;
- }
-
- /**
- *
- */
- public static void init()
- {
- _mQueue = new EvictionQueue();
- _size = 0;
- if( CacheableData.CACHING_BUFFER_PAGECACHE )
- PageCache.init();
- }
-
- /**
- *
- */
- public static void cleanup()
- {
- if( _mQueue!=null )
- _mQueue.clear();
- if( CacheableData.CACHING_BUFFER_PAGECACHE )
- PageCache.clear();
- }
-
- /**
- *
- * @return
- */
- public static long getWriteBufferSize()
- {
- long maxMem = InfrastructureAnalyzer.getLocalMaxMemory();
- return (long)(CacheableData.CACHING_BUFFER_SIZE * maxMem);
- }
-
- /**
- *
- */
- public static void printStatus( String position )
- {
- System.out.println("WRITE BUFFER STATUS ("+position+") --");
-
- //print buffer meta data
- System.out.println("\tWB: Buffer Meta Data: " +
- "limit="+_limit+", " +
- "size[bytes]="+_size+", " +
- "size[elements]="+_mQueue.size()+"/"+_mQueue.size());
-
- //print current buffer entries
- int count = _mQueue.size();
- for( Entry<String, ByteBuffer> entry : _mQueue.entrySet() )
- {
- String fname = entry.getKey();
- ByteBuffer bbuff = entry.getValue();
-
- System.out.println("\tWB: buffer element ("+count+"): "+fname+", "+bbuff.getSize()+", "+bbuff.isInSparseFormat());
- count--;
- }
- }
-
- /**
- * Extended LinkedHashMap with convenience methods for adding and removing
- * last/first entries.
- *
- */
- private static class EvictionQueue extends LinkedHashMap<String, ByteBuffer>
- {
- private static final long serialVersionUID = -5208333402581364859L;
-
- public void addLast( String fname, ByteBuffer bbuff )
- {
- //put entry into eviction queue w/ 'addLast' semantics
- put(fname, bbuff);
- }
-
- public Entry<String, ByteBuffer> removeFirst()
- {
- //move iterator to first entry
- Iterator<Entry<String, ByteBuffer>> iter = entrySet().iterator();
- Entry<String, ByteBuffer> entry = iter.next();
-
- //remove current iterator entry
- iter.remove();
-
- return entry;
- }
- }
-}
[26/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java b/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
deleted file mode 100644
index 68f4b74..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform a partial aggregation. It was introduced to do some initial
- * aggregation operations on blocks in the mapper/reducer.
- */
-
-public class PartialAggregate extends Lop
-{
-
-
- public enum DirectionTypes {
- RowCol,
- Row,
- Col
- };
-
- public enum CorrectionLocationType {
- NONE,
- LASTROW,
- LASTCOLUMN,
- LASTTWOROWS,
- LASTTWOCOLUMNS,
- INVALID
- };
-
- private Aggregate.OperationTypes operation;
- private DirectionTypes direction;
- private boolean _dropCorr = false;
-
- //optional attribute for CP num threads
- private int _numThreads = -1;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, ExecType.MR);
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et, int k)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- _numThreads = k;
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- _aggtype = aggtype;
- }
-
- /**
- * Constructor to setup a partial aggregate operation.
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- private void init(Lop input,
- Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et) {
- operation = op;
- direction = direct;
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR )
- {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, STANDALONE MMCJ:
- * only in mapper.
- */
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- this.lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else //CP | SPARK
- {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public void setDropCorrection()
- {
- _dropCorr = true;
- }
-
- public static CorrectionLocationType decodeCorrectionLocation(String loc)
- {
- return CorrectionLocationType.valueOf(loc);
- }
-
- /**
- * This method computes the location of "correction" terms in the output
- * produced by PartialAgg instruction.
- *
- * When computing the stable sum, "correction" refers to the compensation as
- * defined by the original Kahan algorithm. When computing the stable mean,
- * "correction" refers to two extra values (the running mean, count)
- * produced by each Mapper i.e., by each PartialAgg instruction.
- *
- * This method is invoked during hop-to-lop translation, while creating the
- * corresponding Aggregate lop
- *
- * Computed information is encoded in the PartialAgg instruction so that the
- * appropriate aggregate operator is used at runtime (see:
- * dml.runtime.matrix.operator.AggregateOperator.java and dml.runtime.matrix)
- */
- public CorrectionLocationType getCorrectionLocation()
- throws LopsException
- {
- return getCorrectionLocation(operation, direction);
- }
-
- public static CorrectionLocationType getCorrectionLocation(Aggregate.OperationTypes operation, DirectionTypes direction)
- throws LopsException
- {
- CorrectionLocationType loc;
-
- switch (operation) {
- case KahanSum:
- case KahanSumSq:
- case KahanTrace:
- switch (direction) {
- case Col:
- // colSums: corrections will be present as a last row in the
- // result
- loc = CorrectionLocationType.LASTROW;
- break;
- case Row:
- case RowCol:
- // rowSums, sum: corrections will be present as a last column in
- // the result
- loc = CorrectionLocationType.LASTCOLUMN;
- break;
- default:
- throw new LopsException("PartialAggregate.getCorrectionLocation() - "
- + "Unknown aggregate direction: " + direction);
- }
- break;
-
- case Mean:
- // Computation of stable mean requires each mapper to output both
- // the running mean as well as the count
- switch (direction) {
- case Col:
- // colMeans: last row is correction 2nd last is count
- loc = CorrectionLocationType.LASTTWOROWS;
- break;
- case Row:
- case RowCol:
- // rowMeans, mean: last column is correction 2nd last is count
- loc = CorrectionLocationType.LASTTWOCOLUMNS;
- break;
- default:
- throw new LopsException("PartialAggregate.getCorrectionLocation() - "
- + "Unknown aggregate direction: " + direction);
- }
- break;
-
- case MaxIndex:
- case MinIndex:
- loc = CorrectionLocationType.LASTCOLUMN;
- break;
-
- default:
- loc = CorrectionLocationType.NONE;
- }
- return loc;
- }
-
- public void setDimensionsBasedOnDirection(long dim1, long dim2,
- long rowsPerBlock, long colsPerBlock) throws LopsException
- {
- setDimensionsBasedOnDirection(this, dim1, dim2, rowsPerBlock, colsPerBlock, direction);
- }
-
- public static void setDimensionsBasedOnDirection(Lop lop, long dim1, long dim2,
- long rowsPerBlock, long colsPerBlock, DirectionTypes dir)
- throws LopsException
- {
- try {
- if (dir == DirectionTypes.Row)
- lop.outParams.setDimensions(dim1, 1, rowsPerBlock, colsPerBlock, -1);
- else if (dir == DirectionTypes.Col)
- lop.outParams.setDimensions(1, dim2, rowsPerBlock, colsPerBlock, -1);
- else if (dir == DirectionTypes.RowCol)
- lop.outParams.setDimensions(1, 1, rowsPerBlock, colsPerBlock, -1);
- else
- throw new LopsException("In PartialAggregate Lop, Unknown aggregate direction " + dir);
- } catch (HopsException e) {
- throw new LopsException("In PartialAggregate Lop, error setting dimensions based on direction", e);
- }
- }
-
- public String toString() {
- return "Partial Aggregate " + operation;
- }
-
- private String getOpcode() {
- return getOpcode(operation, direction);
- }
-
- /**
- * Instruction generation for for CP and Spark
- */
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output) );
-
- //in case of spark, we also compile the optional aggregate flag into the instruction.
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
- }
-
- //in case of cp, we also compile the number of threads into the instruction
- if( getExecType() == ExecType.CP ){
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _dropCorr );
-
- return sb.toString();
- }
-
- /**
- *
- * @param op
- * @param dir
- * @return
- */
- public static String getOpcode(Aggregate.OperationTypes op, DirectionTypes dir)
- {
- switch( op )
- {
- case Sum: {
- if( dir == DirectionTypes.RowCol )
- return "ua+";
- else if( dir == DirectionTypes.Row )
- return "uar+";
- else if( dir == DirectionTypes.Col )
- return "uac+";
- break;
- }
-
- case Mean: {
- if( dir == DirectionTypes.RowCol )
- return "uamean";
- else if( dir == DirectionTypes.Row )
- return "uarmean";
- else if( dir == DirectionTypes.Col )
- return "uacmean";
- break;
- }
-
- case KahanSum: {
- // instructions that use kahanSum are similar to ua+,uar+,uac+
- // except that they also produce correction values along with partial
- // sums.
- if( dir == DirectionTypes.RowCol )
- return "uak+";
- else if( dir == DirectionTypes.Row )
- return "uark+";
- else if( dir == DirectionTypes.Col )
- return "uack+";
- break;
- }
-
- case KahanSumSq: {
- if( dir == DirectionTypes.RowCol )
- return "uasqk+";
- else if( dir == DirectionTypes.Row )
- return "uarsqk+";
- else if( dir == DirectionTypes.Col )
- return "uacsqk+";
- break;
- }
-
- case Product: {
- if( dir == DirectionTypes.RowCol )
- return "ua*";
- break;
- }
-
- case Max: {
- if( dir == DirectionTypes.RowCol )
- return "uamax";
- else if( dir == DirectionTypes.Row )
- return "uarmax";
- else if( dir == DirectionTypes.Col )
- return "uacmax";
- break;
- }
-
- case Min: {
- if( dir == DirectionTypes.RowCol )
- return "uamin";
- else if( dir == DirectionTypes.Row )
- return "uarmin";
- else if( dir == DirectionTypes.Col )
- return "uacmin";
- break;
- }
-
- case MaxIndex:{
- if( dir == DirectionTypes.Row )
- return "uarimax";
- break;
- }
-
- case MinIndex: {
- if( dir == DirectionTypes.Row )
- return "uarimin";
- break;
- }
-
- case Trace: {
- if( dir == DirectionTypes.RowCol)
- return "uatrace";
- break;
- }
-
- case KahanTrace: {
- if( dir == DirectionTypes.RowCol )
- return "uaktrace";
- break;
- }
- }
-
- //should never come here for normal compilation
- throw new UnsupportedOperationException("Instruction is not defined for PartialAggregate operation " + op);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PickByCount.java b/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
deleted file mode 100644
index f0cca24..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class PickByCount extends Lop
-{
-
- public static final String OPCODE = "qpick";
-
- public enum OperationTypes {
- VALUEPICK,
- RANGEPICK,
- IQM,
- MEDIAN
- };
-
- private OperationTypes operation;
- private boolean inMemoryInput = false;
-
-
- /*
- * valuepick: first input is always a matrix, second input can either be a scalar or a matrix
- * rangepick: first input is always a matrix, second input is always a scalar
- */
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op) {
- this(input1, input2, dt, vt, op, ExecType.MR);
- }
-
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op, ExecType et) {
- super(Lop.Type.PickValues, dt, vt);
- init(input1, input2, op, et);
- }
-
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op, ExecType et, boolean inMemoryInput) {
- super(Lop.Type.PickValues, dt, vt);
- this.inMemoryInput = inMemoryInput;
- init(input1, input2, op, et);
- }
-
-
- private void init(Lop input1, Lop input2, OperationTypes op, ExecType et) {
- this.addInput(input1);
- input1.addOutput(this);
-
- if ( input2 != null ) {
- this.addInput(input2);
- input2.addOutput(this);
- }
-
- operation = op;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, et, ExecLocation.RecordReader, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return "Operation: " + operation;
- }
-
- public OperationTypes getOperationType() {
- return operation;
- }
-
- /*
- * This version of getInstruction() must be called only for valuepick (MR) and rangepick
- *
- * Example instances:
- * valupick:::0:DOUBLE:::1:DOUBLE:::2:DOUBLE
- * rangepick:::0:DOUBLE:::0.25:DOUBLE:::1:DOUBLE
- * rangepick:::0:DOUBLE:::Var3:DOUBLE:::1:DOUBLE
- */
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- return getInstructions(""+input_index1, ""+input_index2, ""+output_index);
-
- }
-
- /*
- * This version of getInstructions() must be called only for valuepick (CP), IQM (CP)
- *
- * Example instances:
- * valuepick:::temp2:STRING:::0.25:DOUBLE:::Var1:DOUBLE
- * valuepick:::temp2:STRING:::Var1:DOUBLE:::Var2:DOUBLE
- */
- @Override
- public String getInstructions(String input1, String input2, String output) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- if(operation != OperationTypes.MEDIAN) {
- if ( getInputs().get(1).getDataType() == DataType.SCALAR )
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
- }
-
- sb.append( this.prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(operation);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(inMemoryInput);
-
- return sb.toString();
- }
-
- /**
- * This version of getInstructions() is called for IQM, executing in CP
- *
- * Example instances:
- * iqm:::input:::output
- */
- @Override
- public String getInstructions(String input, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output) );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(operation);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(inMemoryInput);
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java b/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
deleted file mode 100644
index a36c0d3..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class RangeBasedReIndex extends Lop
-{
-
- /**
- * Constructor to setup a RangeBasedReIndex operation.
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
-
- private boolean forLeftIndexing = false;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et, boolean forleft)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- _aggtype = aggtype;
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim,
- Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft)
- {
- addInput(inputMatrix);
- addInput(rowL);
- addInput(rowU);
- addInput(colL);
- addInput(colU);
- addInput(leftMatrixRowDim);
- addInput(leftMatrixColDim);
-
- inputMatrix.addOutput(this);
- rowL.addOutput(this);
- rowU.addOutput(this);
- colL.addOutput(this);
- colU.addOutput(this);
- leftMatrixRowDim.addOutput(this);
- leftMatrixColDim.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- forLeftIndexing=forleft;
- }
-
- private String getOpcode() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
- @Override
- public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output)
- throws LopsException {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append( getInputs().get(1).prepScalarInputOperand(rowl));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(rowu));
- sb.append( OPERAND_DELIMITOR );
-
- // coll, colu
- sb.append( getInputs().get(3).prepScalarInputOperand(coll));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(4).prepScalarInputOperand(colu));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( output );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
-
- if(getExecType() == ExecType.MR) {
- // following fields are added only when this lop is executed in MR (both for left & right indexing)
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim));
- }
-
- //in case of spark, we also compile the optional aggregate flag into the instruction.
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index)
- throws LopsException {
- /*
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
- */
- String rowl = getInputs().get(1).prepScalarLabel();
- String rowu = getInputs().get(2).prepScalarLabel();
- String coll = getInputs().get(3).prepScalarLabel();
- String colu = getInputs().get(4).prepScalarLabel();
-
- String left_nrow = getInputs().get(5).prepScalarLabel();
- String left_ncol = getInputs().get(6).prepScalarLabel();
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index));
- }
-
- @Override
- public String toString() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/ReBlock.java b/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
deleted file mode 100644
index a442fa6..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform reblock operation
- */
-public class ReBlock extends Lop
-{
-
- public static final String OPCODE = "rblk";
-
- private boolean _outputEmptyBlocks = true;
-
- /**
- * Constructor to perform a reblock operation.
- * @param input
- * @param op
- */
-
- private Long _rows_per_block;
- private Long _cols_per_block;
-
- public ReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, boolean outputEmptyBlocks, ExecType et) throws LopsException
- {
- super(Lop.Type.ReBlock, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- _rows_per_block = rows_per_block;
- _cols_per_block = cols_per_block;
-
- _outputEmptyBlocks = outputEmptyBlocks;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- lps.addCompatibility(JobType.REBLOCK);
-
- if(et == ExecType.MR)
- lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- else if(et == ExecType.SPARK)
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- else
- throw new LopsException("Incorrect execution type for Reblock:" + et);
- }
-
- @Override
- public String toString() {
-
- return "Reblock - rows per block = " + _rows_per_block + " cols per block " + _cols_per_block ;
- }
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( this.prepOutputOperand(output_index));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _rows_per_block );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cols_per_block );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("The method getInstructions(String,String) for Reblock should be called only for Spark execution type");
- }
-
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "rblk" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( _rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cols_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for Reblock operation");
- }
- }
-
- // This function is replicated in Dag.java
- @SuppressWarnings("unused")
- private Format getChildFormat(Lop node) throws LopsException {
-
- if(node.getOutputParameters().getFile_name() != null
- || node.getOutputParameters().getLabel() != null)
- {
- return node.getOutputParameters().getFormat();
- }
- else
- {
- // Reblock lop should always have a single child
- if(node.getInputs().size() > 1)
- throw new LopsException(this.printErrorLocation() + "Should only have one child! \n");
-
- /*
- * Return the format of the child node (i.e., input lop)
- * No need of recursion here.. because
- * 1) Reblock lop's input can either be DataLop or some intermediate computation
- * If it is Data then we just take its format (TEXT or BINARY)
- * If it is intermediate lop then it is always BINARY
- * since we assume that all intermediate computations will be in Binary format
- * 2) Note that Reblock job will never have any instructions in the mapper
- * => the input lop (if it is other than Data) is always executed in a different job
- */
- // return getChildFormat(node.getInputs().get(0));
- return node.getInputs().get(0).getOutputParameters().getFormat(); }
-
- }
-
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/RepMat.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/RepMat.java b/src/main/java/com/ibm/bi/dml/lops/RepMat.java
deleted file mode 100644
index b838539..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/RepMat.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class RepMat extends Lop
-{
-
- public static final String OPCODE = "rep";
-
- private boolean _repCols = true;
-
- /**
- * Constructor to setup a partial Matrix-Vector Multiplication
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public RepMat(Lop input1, Lop input2, boolean repCols, DataType dt, ValueType vt)
- throws LopsException
- {
- super(Lop.Type.RepMat, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- _repCols = repCols;
-
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
-
-
- public String toString() {
- return "Operation = RepMat";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_repCols);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/SortKeys.java b/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
deleted file mode 100644
index 39b0ea7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-public class SortKeys extends Lop
-{
-
- public static final String OPCODE = "qsort"; //quantile sort
-
- public enum OperationTypes {
- WithWeights,
- WithoutWeights,
- Indexes,
- };
-
- private OperationTypes operation;
- private boolean descending = false;
-
- public OperationTypes getOpType() {
- return operation;
- }
-
- public SortKeys(Lop input, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, ExecType.MR);
- }
-
- public SortKeys(Lop input, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, et);
- }
-
- public SortKeys(Lop input, boolean desc, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, et);
- descending = desc;
- }
-
- public SortKeys(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input1, input2, op, et);
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, ExecType et) {
- this.addInput(input1);
- input1.addOutput(this);
-
- operation = op;
-
- if ( et == ExecType.MR ) {
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- lps.addCompatibility(JobType.SORT);
- this.lps.setProperties( inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- if(op != OperationTypes.Indexes)
- this.lps.setProducesIntermediateOutput(true);
- }
- else {
- // SortKeys can accept a optional second input only when executing in CP
- // Example: sorting with weights inside CP
- if ( input2 != null ) {
- this.addInput(input2);
- input2.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, false, false, false);
- }
- }
-
-
- @Override
- public String toString() {
- return "Operation: SortKeys (" + operation + ")";
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- {
- return getInstructions(String.valueOf(input_index), String.valueOf(output_index));
- }
-
- @Override
- public String getInstructions(String input, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
- sb.append ( this.prepOutputOperand(output));
-
- if( getExecType() == ExecType.MR ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( operation );
- sb.append( OPERAND_DELIMITOR );
- sb.append( descending );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- // This method is invoked in two cases:
- // 1) SortKeys (both weighted and unweighted) executes in MR
- // 2) Unweighted SortKeys executes in CP
- public static SortKeys constructSortByValueLop(Lop input1, OperationTypes op,
- DataType dt, ValueType vt, ExecType et) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.SortKeys ) {
- return (SortKeys)lop;
- }
- }
-
- SortKeys retVal = new SortKeys(input1, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
- // This method is invoked ONLY for the case of Weighted SortKeys executing in CP
- public static SortKeys constructSortByValueLop(Lop input1, Lop input2, OperationTypes op,
- DataType dt, ValueType vt, ExecType et) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
- // find intersection of input1.getOutputs() and input2.getOutputs();
- set1.retainAll(input2.getOutputs());
-
- for (Lop lop : set1) {
- if ( lop.type == Lop.Type.SortKeys ) {
- return (SortKeys)lop;
- }
- }
-
- SortKeys retVal = new SortKeys(input1, input2, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Ternary.java b/src/main/java/com/ibm/bi/dml/lops/Ternary.java
deleted file mode 100644
index 2feeaac..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Ternary.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform tertiary operation. All inputs must be matrices or vectors.
- * For example, this lop is used in evaluating A = ctable(B,C,W)
- *
- * Currently, this lop is used only in case of CTABLE functionality.
- */
-
-public class Ternary extends Lop
-{
-
- private boolean _ignoreZeros = false;
-
- public enum OperationTypes {
- CTABLE_TRANSFORM,
- CTABLE_TRANSFORM_SCALAR_WEIGHT,
- CTABLE_TRANSFORM_HISTOGRAM,
- CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM,
- CTABLE_EXPAND_SCALAR_WEIGHT,
- INVALID };
-
- OperationTypes operation;
-
-
- public Ternary(Lop[] inputLops, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- this(inputLops, op, dt, vt, false, et);
- }
-
- public Ternary(Lop[] inputLops, OperationTypes op, DataType dt, ValueType vt, boolean ignoreZeros, ExecType et) {
- super(Lop.Type.Ternary, dt, vt);
- init(inputLops, op, et);
- _ignoreZeros = ignoreZeros;
- }
-
- private void init(Lop[] inputLops, OperationTypes op, ExecType et) {
- operation = op;
-
- for(int i=0; i < inputLops.length; i++) {
- this.addInput(inputLops[i]);
- inputLops[i].addOutput(this);
- }
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- //lps.addCompatibility(JobType.DATAGEN); MB: disabled due to piggybacking issues
- //lps.addCompatibility(JobType.REBLOCK); MB: disabled since no runtime support
-
- if( operation==OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT )
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- //TODO create runtime for ctable in gmr mapper and switch to maporreduce.
- //this.lps.setProperties( inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- else
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
-
- }
-
- public static OperationTypes findCtableOperationByInputDataTypes(DataType dt1, DataType dt2, DataType dt3)
- {
- if ( dt1 == DataType.MATRIX ) {
- if (dt2 == DataType.MATRIX && dt3 == DataType.SCALAR) {
- // F = ctable(A,B) or F = ctable(A,B,1)
- return OperationTypes.CTABLE_TRANSFORM_SCALAR_WEIGHT;
- } else if (dt2 == DataType.SCALAR && dt3 == DataType.SCALAR) {
- // F=ctable(A,1) or F = ctable(A,1,1)
- return OperationTypes.CTABLE_TRANSFORM_HISTOGRAM;
- } else if (dt2 == DataType.SCALAR && dt3 == DataType.MATRIX) {
- // F=ctable(A,1,W)
- return OperationTypes.CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM;
- } else {
- // F=ctable(A,B,W)
- return OperationTypes.CTABLE_TRANSFORM;
- }
- }
- else {
- return OperationTypes.INVALID;
- }
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- if( operation != Ternary.OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT )
- sb.append( "ctable" );
- else
- sb.append( "ctableexpand" );
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(0).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(0).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(0).prepInputOperand(input1));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(1).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(2).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(2).prepInputOperand(input3));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( this.getInputs().size() > 3 ) {
- sb.append(getInputs().get(3).getOutputParameters().getLabel());
- sb.append(LITERAL_PREFIX);
- sb.append((getInputs().get(3).getType() == Type.Data && ((Data)getInputs().get(3)).isLiteral()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(4).getOutputParameters().getLabel());
- sb.append(LITERAL_PREFIX);
- sb.append((getInputs().get(4).getType() == Type.Data && ((Data)getInputs().get(4)).isLiteral()) );
- sb.append( OPERAND_DELIMITOR );
- }
- else {
- sb.append(-1);
- sb.append(LITERAL_PREFIX);
- sb.append(true);
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(-1);
- sb.append(LITERAL_PREFIX);
- sb.append(true);
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _ignoreZeros );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- switch(operation) {
- /* Arithmetic */
- case CTABLE_TRANSFORM:
- // F = ctable(A,B,W)
- sb.append( "ctabletransform" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_SCALAR_WEIGHT:
- // F = ctable(A,B) or F = ctable(A,B,1)
- // third input must be a scalar, and hence input_index3 == -1
- if ( input_index3 != -1 ) {
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
- }
-
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
- int scalarIndex = 2; // index of the scalar input
-
- sb.append( "ctabletransformscalarweight" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_EXPAND_SCALAR_WEIGHT:
- // F = ctable(seq,B) or F = ctable(seq,B,1)
- // second and third inputs must be scalars, and hence input_index2 == -1, input_index3 == -1
- if ( input_index3 != -1 ) {
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
- }
-
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
- int scalarIndex2 = 1; // index of the scalar input
- int scalarIndex3 = 2; // index of the scalar input
-
- sb.append( "ctableexpandscalarweight" );
- sb.append( OPERAND_DELIMITOR );
- //get(0) because input under group
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex2).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex3).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_HISTOGRAM:
- // F=ctable(A,1) or F = ctable(A,1,1)
- if ( input_index2 != -1 || input_index3 != -1)
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation);
-
- // 2nd and 3rd inputs are scalar inputs
-
- sb.append( "ctabletransformhistogram" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM:
- // F=ctable(A,1,W)
- if ( input_index2 != -1 )
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation);
-
- // 2nd input is the scalar input
-
- sb.append( "ctabletransformweightedhistogram" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Tertiary operation: " + operation);
- }
-
- long outputDim1=-1, outputDim2=-1;
- if ( getInputs().size() > 3 ) {
- sb.append(getInputs().get(3).prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(4).prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
- /*if ( input3 instanceof Data && ((Data)input3).isLiteral()
- && input4 instanceof Data && ((Data)input4).isLiteral() ) {
- outputDim1 = ((Data)input3).getLongValue();
- outputDim2 = ((Data)input4).getLongValue();
- }*/
- }
- else {
- sb.append( outputDim1 );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( outputDim2 );
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- /**
- *
- * @param type
- * @return
- */
- public static String getOpcode(OperationTypes type)
- {
- switch( type )
- {
- case CTABLE_TRANSFORM: return "ctabletransform";
- case CTABLE_TRANSFORM_SCALAR_WEIGHT: return "ctabletransformscalarweight";
- case CTABLE_EXPAND_SCALAR_WEIGHT: return "ctableexpandscalarweight";
- case CTABLE_TRANSFORM_HISTOGRAM: return "ctabletransformhistogram";
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM: return "ctabletransformweightedhistogram";
-
- default:
- throw new UnsupportedOperationException("Ternary operation code is not defined: " + type);
- }
- }
-
- /**
- *
- * @param opcode
- * @return
- */
- public static OperationTypes getOperationType(String opcode)
- {
- OperationTypes op = null;
-
- if( opcode.equals("ctabletransform") )
- op = OperationTypes.CTABLE_TRANSFORM;
- else if( opcode.equals("ctabletransformscalarweight") )
- op = OperationTypes.CTABLE_TRANSFORM_SCALAR_WEIGHT;
- else if( opcode.equals("ctableexpandscalarweight") )
- op = OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT;
- else if( opcode.equals("ctabletransformhistogram") )
- op = OperationTypes.CTABLE_TRANSFORM_HISTOGRAM;
- else if( opcode.equals("ctabletransformweightedhistogram") )
- op = OperationTypes.CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM;
- else
- throw new UnsupportedOperationException("Tertiary operation code is not defined: " + opcode);
-
- return op;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java b/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
deleted file mode 100644
index 2687b98..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- *
- *
- */
-public class TernaryAggregate extends Lop
-{
-
- private static final String OPCODE = "tak+*";
-
- //NOTE: currently only used for ta+*
- //private Aggregate.OperationTypes _aggOp = null;
- //private Binary.OperationTypes _binOp = null;
-
- //optional attribute for cp
- private int _numThreads = -1;
-
- public TernaryAggregate(Lop input1, Lop input2, Lop input3, Aggregate.OperationTypes aggOp, Binary.OperationTypes binOp, DataType dt, ValueType vt, ExecType et ) {
- this(input1, input2, input3, aggOp, binOp, dt, vt, et, 1);
- }
-
- /**
- * @param et
- * @param input - input lop
- * @param op - operation type
- */
- public TernaryAggregate(Lop input1, Lop input2, Lop input3, Aggregate.OperationTypes aggOp, Binary.OperationTypes binOp, DataType dt, ValueType vt, ExecType et, int k )
- {
- super(Lop.Type.TernaryAggregate, dt, vt);
-
- //_aggOp = aggOp;
- //_binOp = binOp;
-
- addInput(input1);
- addInput(input2);
- addInput(input3);
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- _numThreads = k;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString()
- {
- return "Operation: "+OPCODE;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepInputOperand(input3));
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
-
- if( getExecType() == ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Transform.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Transform.java b/src/main/java/com/ibm/bi/dml/lops/Transform.java
deleted file mode 100644
index b4e6d13..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Transform.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/*
- * Lop to perform transpose/vector to diag operations
- * This lop can change the keys and hence break alignment.
- */
-
-public class Transform extends Lop
-{
-
-
- public enum OperationTypes {
- Transpose,
- Diag,
- Reshape,
- Sort
- };
-
- private boolean _bSortIndInMem = false;
-
- private OperationTypes operation = null;
-
- /**
- * Constructor when we have one input.
- * @param input
- * @param op
- */
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- super(Lop.Type.Transform, dt, vt);
- init(input, op, dt, vt, et);
- }
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.Transform, dt, vt);
- init(input, op, dt, vt, ExecType.MR);
- }
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean bSortIndInMem)
- {
- super(Lop.Type.Transform, dt, vt);
- _bSortIndInMem = bSortIndInMem;
- init(input, op, dt, vt, et);
- }
-
- private void init (Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- operation = op;
-
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- if ( et == ExecType.MR ) {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, STANDALONE
- * MMCJ: only in mapper.
- */
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
-
- if( op == OperationTypes.Reshape )
- //reshape should be executed in map because we have potentially large intermediate data and want to exploit the combiner.
- this.lps.setProperties( inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- else
- this.lps.setProperties( inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- }
- else //CP/SPARK
- {
- // <code>breaksAlignment</code> is not meaningful when <code>Transform</code> executes in CP.
- breaksAlignment = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- private String getOpcode() {
- switch(operation) {
- case Transpose:
- // Transpose a matrix
- return "r'";
-
- case Diag:
- // Transform a vector into a diagonal matrix
- return "rdiag";
-
- case Reshape:
- // Transform a vector into a diagonal matrix
- return "rshape";
-
- case Sort:
- // Transform a matrix into a sorted matrix
- return "rsort";
-
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Transform operation " + operation);
-
- }
- }
-
- //CP instructions
-
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String input4, String output)
- throws LopsException
- {
- //only used for reshape
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- //rows, cols, byrow
- String[] inputX = new String[]{input2,input3,input4};
- for( int i=1; i<=(inputX.length); i++ ) {
- Lop ltmp = getInputs().get(i);
- sb.append( OPERAND_DELIMITOR );
- sb.append( ltmp.prepScalarInputOperand(getExecType()));
- }
-
- //output
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- if( getExecType()==ExecType.SPARK && operation == OperationTypes.Sort ){
- sb.append( OPERAND_DELIMITOR );
- sb.append( _bSortIndInMem);
- }
-
- return sb.toString();
- }
-
- //MR instructions
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index)
- throws LopsException
- {
- //only used for reshape
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- //rows
- Lop input2 = getInputs().get(1);
- String rowsString = input2.prepScalarLabel();
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsString );
-
- //cols
- Lop input3 = getInputs().get(2);
- String colsString = input3.prepScalarLabel();
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsString );
-
- //byrow
- Lop input4 = getInputs().get(3);
- String byrowString = input4.prepScalarLabel();
- if ( input4.getExecLocation() == ExecLocation.Data
- && !((Data)input4).isLiteral() || !(input4.getExecLocation() == ExecLocation.Data ) ){
- throw new LopsException(this.printErrorLocation() + "Parameter 'byRow' must be a literal for a matrix operation.");
- }
- sb.append( OPERAND_DELIMITOR );
- sb.append( byrowString );
-
- //output
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java b/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
deleted file mode 100644
index fd36c9b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * TODO Currently this lop only support the right hand side in distributed cache. This
- * should be generalized (incl hop operator selection) to left/right cache types.
- *
- *
- */
-public class UAggOuterChain extends Lop
-{
-
- public static final String OPCODE = "uaggouterchain";
-
- //outer operation
- private Aggregate.OperationTypes _uaggOp = null;
- private PartialAggregate.DirectionTypes _uaggDir = null;
- //inner operation
- private Binary.OperationTypes _binOp = null;
-
-
- /**
- * Constructor to setup a unaryagg outer chain
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public UAggOuterChain(Lop input1, Lop input2, Aggregate.OperationTypes uaop, PartialAggregate.DirectionTypes uadir, Binary.OperationTypes bop, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.UaggOuterChain, dt, vt);
- addInput(input1);
- addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup operator types
- _uaggOp = uaop;
- _uaggDir = uadir;
- _binOp = bop;
-
- //setup MR parameters
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
-
- public String toString() {
- return "Operation = UaggOuterChain";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input2));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- return new int[]{2};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Unary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Unary.java b/src/main/java/com/ibm/bi/dml/lops/Unary.java
deleted file mode 100644
index b29e822..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Unary.java
+++ /dev/null
@@ -1,445 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform following operations: with one operand -- NOT(A), ABS(A),
- * SQRT(A), LOG(A) with two operands where one of them is a scalar -- H=H*i,
- * H=H*5, EXP(A,2), LOG(A,2)
- *
- */
-
-public class Unary extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, SUBTRACTRIGHT, MULTIPLY, MULTIPLY2, DIVIDE, MODULUS, INTDIV, MINUS1_MULTIPLY, POW, POW2, LOG, MAX, MIN, NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, EXP, Over, LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS, ROUND, CEIL, FLOOR, MR_IQM, INVERSE,
- CUMSUM, CUMPROD, CUMMIN, CUMMAX,
- SPROP, SIGMOID, SELP, SUBTRACT_NZ, LOG_NZ,
- NOTSUPPORTED
- };
-
- OperationTypes operation;
-
- Lop valInput;
-
- /**
- * Constructor to perform a unary operation with 2 inputs
- *
- * @param input
- * @param op
- */
-
- public Unary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, input2, op, dt, vt, et);
- }
-
- public Unary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, input2, op, dt, vt, ExecType.MR);
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- operation = op;
-
- if (input1.getDataType() == DataType.MATRIX)
- valInput = input2;
- else
- valInput = input1;
-
- this.addInput(input1);
- input1.addOutput(this);
- this.addInput(input2);
- input2.addOutput(this);
-
- // By definition, this lop should not break alignment
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, CM_COV, and COMBINE
- * jobs MMCJ: only in mapper.
- */
- lps.addCompatibility(JobType.ANY);
- lps.removeNonPiggybackableJobs();
- lps.removeCompatibility(JobType.CM_COV); // CM_COV allows only reducer instructions but this is MapOrReduce. TODO: piggybacking should be updated to take this extra constraint.
- lps.removeCompatibility(JobType.TRANSFORM);
- this.lps.setProperties(inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- /**
- * Constructor to perform a unary operation with 1 input.
- *
- * @param input1
- * @param op
- */
- public Unary(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, op, dt, vt, et);
- }
-
- public Unary(Lop input1, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, op, dt, vt, ExecType.MR);
- }
-
- private ExecType forceExecType(OperationTypes op, ExecType et) {
- if ( op == OperationTypes.INVERSE )
- return ExecType.CP;
- return et;
- }
- private void init(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- operation = op;
-
- et = forceExecType(op, et);
-
- valInput = null;
-
- this.addInput(input1);
- input1.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- /*
- * This lop can be executed in all jobs except for PARTITION. MMCJ: only
- * in mapper. GroupedAgg: only in reducer.
- */
- lps.addCompatibility(JobType.ANY);
- lps.removeNonPiggybackableJobs();
- lps.removeCompatibility(JobType.CM_COV); // CM_COV allows only reducer instructions but this is MapOrReduce. TODO: piggybacking should be updated to take this extra constraint.
- lps.removeCompatibility(JobType.TRANSFORM);
- this.lps.setProperties(inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- @Override
- public String toString() {
- if (valInput != null)
- return "Operation: " + operation + " " + "Label: "
- + valInput.getOutputParameters().getLabel()
- + " input types " + this.getInputs().get(0).toString()
- + " " + this.getInputs().get(1).toString();
- else
- return "Operation: " + operation + " " + "Label: N/A";
- }
-
- /**
- *
- * @return
- * @throws LopsException
- */
- private String getOpcode()
- throws LopsException
- {
- return getOpcode(operation);
- }
-
- /**
- *
- * @param op
- * @return
- * @throws LopsException
- */
- public static String getOpcode(OperationTypes op)
- throws LopsException
- {
- switch (op) {
- case NOT:
- return "!";
- case ABS:
- return "abs";
- case SIN:
- return "sin";
- case COS:
- return "cos";
- case TAN:
- return "tan";
- case ASIN:
- return "asin";
- case ACOS:
- return "acos";
- case ATAN:
- return "atan";
- case SQRT:
- return "sqrt";
- case EXP:
- return "exp";
-
- case LOG:
- return "log";
-
- case LOG_NZ:
- return "log_nz";
-
- case ROUND:
- return "round";
-
- case ADD:
- return "+";
-
- case SUBTRACT:
- return "-";
-
- case SUBTRACT_NZ:
- return "-nz";
-
- case SUBTRACTRIGHT:
- return "s-r";
-
- case MULTIPLY:
- return "*";
-
- case MULTIPLY2:
- return "*2";
-
- case MINUS1_MULTIPLY:
- return "1-*";
-
- case DIVIDE:
- return "/";
-
- case MODULUS:
- return "%%";
-
- case INTDIV:
- return "%/%";
-
- case Over:
- return "so";
-
- case POW:
- return "^";
-
- case POW2:
- return "^2";
-
- case GREATER_THAN:
- return ">";
-
- case GREATER_THAN_OR_EQUALS:
- return ">=";
-
- case LESS_THAN:
- return "<";
-
- case LESS_THAN_OR_EQUALS:
- return "<=";
-
- case EQUALS:
- return "==";
-
- case NOT_EQUALS:
- return "!=";
-
- case MAX:
- return "max";
-
- case MIN:
- return "min";
-
- case CEIL:
- return "ceil";
-
- case FLOOR:
- return "floor";
-
- case CUMSUM:
- return "ucumk+";
-
- case CUMPROD:
- return "ucum*";
-
- case CUMMIN:
- return "ucummin";
-
- case CUMMAX:
- return "ucummax";
-
- case INVERSE:
- return "inverse";
-
- case MR_IQM:
- return "qpick";
-
- case SPROP:
- return "sprop";
-
- case SIGMOID:
- return "sigmoid";
-
- case SELP:
- return "sel+";
-
- default:
- throw new LopsException(
- "Instruction not defined for Unary operation: " + op);
- }
- }
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- // Unary operators with one input
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for an Unary opration: "
- + operation);
- }
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException {
- return getInstructions(""+input_index, ""+output_index);
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(0).getDataType() == DataType.SCALAR ) {
- sb.append( getInputs().get(0).prepScalarInputOperand(getExecType()));
- }
- else {
- sb.append( getInputs().get(0).prepInputOperand(input1));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(1).getDataType() == DataType.SCALAR ) {
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- }
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int inputIndex1, int inputIndex2,
- int outputIndex) throws LopsException {
- if (this.getInputs().size() == 2) {
- // Unary operators with two inputs
- // Determine the correct operation, depending on the scalar input
- Lop linput1 = getInputs().get(0);
- Lop linput2 = getInputs().get(1);
-
- int scalarIndex = -1, matrixIndex = -1;
- String matrixLabel= null;
- if( linput1.getDataType() == DataType.MATRIX ) {
- // inputIndex1 is matrix, and inputIndex2 is scalar
- scalarIndex = 1;
- matrixLabel = String.valueOf(inputIndex1);
- }
- else {
- // inputIndex2 is matrix, and inputIndex1 is scalar
- scalarIndex = 0;
- matrixLabel = String.valueOf(inputIndex2);
-
- // when the first operand is a scalar, setup the operation type accordingly
- if (operation == OperationTypes.SUBTRACT)
- operation = OperationTypes.SUBTRACTRIGHT;
- else if (operation == OperationTypes.DIVIDE)
- operation = OperationTypes.Over;
- }
- matrixIndex = 1-scalarIndex;
-
- // Prepare the instruction
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- if( operation == OperationTypes.INTDIV || operation == OperationTypes.MODULUS ||
- operation == OperationTypes.POW ||
- operation == OperationTypes.GREATER_THAN || operation == OperationTypes.GREATER_THAN_OR_EQUALS ||
- operation == OperationTypes.LESS_THAN || operation == OperationTypes.LESS_THAN_OR_EQUALS ||
- operation == OperationTypes.EQUALS || operation == OperationTypes.NOT_EQUALS )
- {
- //TODO discuss w/ Shirish: we should consolidate the other operations (see ScalarInstruction.parseInstruction / BinaryCPInstruction.getScalarOperator)
- //append both operands
- sb.append( (linput1.getDataType()==DataType.MATRIX? linput1.prepInputOperand(String.valueOf(inputIndex1)) : linput1.prepScalarInputOperand(getExecType())) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( (linput2.getDataType()==DataType.MATRIX? linput2.prepInputOperand(String.valueOf(inputIndex2)) : linput2.prepScalarInputOperand(getExecType())) );
- sb.append( OPERAND_DELIMITOR );
- }
- else
- {
- // append the matrix operand
- sb.append( getInputs().get(matrixIndex).prepInputOperand(matrixLabel));
- sb.append( OPERAND_DELIMITOR );
-
- // append the scalar operand
- sb.append( getInputs().get(scalarIndex).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(outputIndex+""));
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for an Unary opration: "
- + operation);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java b/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
deleted file mode 100644
index c92fbfc..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform unary scalar operations. Example a = !b
- *
- */
-
-public class UnaryCP extends Lop
-{
-
- public enum OperationTypes {
- NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, NROW, NCOL, LENGTH, ROUND, STOP, CEIL, FLOOR, CUMSUM, NOTSUPPORTED
- };
-
- public static final String CAST_AS_SCALAR_OPCODE = "castdts";
- public static final String CAST_AS_MATRIX_OPCODE = "castdtm";
- public static final String CAST_AS_DOUBLE_OPCODE = "castvtd";
- public static final String CAST_AS_INT_OPCODE = "castvti";
- public static final String CAST_AS_BOOLEAN_OPCODE = "castvtb";
-
-
-
- OperationTypes operation;
-
- /**
- * Constructor to perform a scalar operation
- *
- * @param input
- * @param op
- */
-
- public UnaryCP(Lop input, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UnaryCP, dt, vt);
- operation = op;
- this.addInput(input);
- input.addOutput(this);
-
- //This lop is executed in control program.
- boolean breaksAlignment = false; // this does not carry any information
- // for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- @Override
- public String toString() {
-
- return "Operation: " + operation;
-
- }
-
- private String getOpCode() throws LopsException {
- switch (operation) {
- case NOT:
- return "!";
-
- case ABS:
- return "abs";
-
- case SIN:
- return "sin";
-
- case COS:
- return "cos";
-
- case TAN:
- return "tan";
-
- case ASIN:
- return "asin";
-
- case ACOS:
- return "acos";
-
- case ATAN:
- return "atan";
-
- case SQRT:
- return "sqrt";
-
- case LOG:
- return "log";
-
- case ROUND:
- return "round";
-
- case EXP:
- return "exp";
-
- case PRINT:
- return "print";
-
- case CAST_AS_MATRIX:
- return CAST_AS_MATRIX_OPCODE;
-
- case STOP:
- return "stop";
-
- case CEIL:
- return "ceil";
-
- case FLOOR:
- return "floor";
-
- case CUMSUM:
- return "ucumk+";
-
- // CAST_AS_SCALAR, NROW, NCOL, LENGTH builtins take matrix as the input
- // and produces a scalar
- case CAST_AS_SCALAR:
- return CAST_AS_SCALAR_OPCODE;
-
- case CAST_AS_DOUBLE:
- return CAST_AS_DOUBLE_OPCODE;
-
- case CAST_AS_INT:
- return CAST_AS_INT_OPCODE;
-
- case CAST_AS_BOOLEAN:
- return CAST_AS_BOOLEAN_OPCODE;
-
- case NROW:
- return "nrow";
-
- case NCOL:
- return "ncol";
-
- case LENGTH:
- return "length";
-
- default:
- throw new LopsException(this.printErrorLocation() + "Unknown operation: " + operation);
- }
- }
-
- @Override
- public String getInstructions(String input, String output)
- throws LopsException {
- StringBuilder sb = new StringBuilder();
- sb.append(getExecType());
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpCode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(0).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
-
- }
-}
[59/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java
index 9cf0bcf..7e3d9e0 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class GNMFTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java b/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java
index 7141b76..33e1b71 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class HITSTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
index 08dca3b..fb3e016 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,11 +26,11 @@ import java.util.List;
import org.junit.Assert;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public abstract class ID3Test extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java
index ef4a712..bfe4b97 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class L2SVMTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/LinearLogRegTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/LinearLogRegTest.java b/src/test/java/org/apache/sysml/test/integration/applications/LinearLogRegTest.java
index 537943e..9abcc28 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/LinearLogRegTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/LinearLogRegTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class LinearLogRegTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java b/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java
index 8424e6f..2e71956 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class LinearRegressionTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java
index a7292bc..97c54cb 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,10 +25,10 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class MDABivariateStatsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java
index 50ddc61..5c838b2 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class MultiClassSVMTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java
index 0d8d932..ee6d215 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,9 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class NaiveBayesTest extends AutomatedTestBase{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java b/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java
index 6876c21..268c149 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -24,9 +24,9 @@ import java.util.List;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public abstract class PageRankTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java b/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java
index 42e5d25..a8531d6 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications;
+package org.apache.sysml.test.integration.applications;
import java.util.ArrayList;
import java.util.Arrays;
@@ -27,10 +27,10 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
@RunWith(value = Parameterized.class)
public abstract class WelchTTest extends AutomatedTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java
index bf692ae..4aae9c7 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class BivariateCategoricalCategoricallTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java
index e361a4f..39d47bb 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class BivariateOrdinalOrdinalTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java
index df4c188..687eb2f 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class BivariateScaleCategoricalTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java
index 8094668..0b24085 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class BivariateScaleScaleTest extends AutomatedTestBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java
index 36508ce..f86ce80 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.Arrays;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class OrderStatisticsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java
index 98b2dad..d199702 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/** Tests of univariate statistics functions over categorical data. */
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java
index bc91178..b1830fa 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java
@@ -1,12 +1,12 @@
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Shared methods and fields from the different univariate stats test harness
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleDenseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleDenseTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleDenseTest.java
index e6a706e..039e600 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleDenseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleDenseTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
/**
* Tests of univariate statistics built-in functions.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleSparseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleSparseTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleSparseTest.java
index 2a98ef4..b0ac313 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleSparseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateUnweightedScaleSparseTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleDenseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleDenseTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleDenseTest.java
index 2a46763..8ae2b56 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleDenseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleDenseTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
/**
* Tests of univariate statistics built-in functions.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleSparseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleSparseTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleSparseTest.java
index 62af05a..d21dbdf 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleSparseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateWeightedScaleSparseTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.descriptivestats;
+package org.apache.sysml.test.integration.applications.descriptivestats;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
/**
* Tests of univariate statistics built-in functions.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/ApplyTransformDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ApplyTransformDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ApplyTransformDMLTest.java
index 537e6aa..c1e2114 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/ApplyTransformDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ApplyTransformDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ApplyTransformTest;
+import org.apache.sysml.test.integration.applications.ApplyTransformTest;
@RunWith(value = Parameterized.class)
public class ApplyTransformDMLTest extends ApplyTransformTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/ArimaDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ArimaDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ArimaDMLTest.java
index 6a08eda..17dc19e 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/ArimaDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ArimaDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ArimaTest;
+import org.apache.sysml.test.integration.applications.ArimaTest;
@RunWith(value = Parameterized.class)
public class ArimaDMLTest extends ArimaTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineCGDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineCGDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineCGDMLTest.java
index 3a6c645..06983f3 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineCGDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineCGDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.CsplineCGTest;
+import org.apache.sysml.test.integration.applications.CsplineCGTest;
@RunWith(value = Parameterized.class)
public class CsplineCGDMLTest extends CsplineCGTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineDSDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineDSDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineDSDMLTest.java
index f974565..5fb3e27 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineDSDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/CsplineDSDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.CsplineDSTest;
+import org.apache.sysml.test.integration.applications.CsplineDSTest;
@RunWith(value = Parameterized.class)
public class CsplineDSDMLTest extends CsplineDSTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/GLMDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/GLMDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/GLMDMLTest.java
index 9a87181..4e7b3be 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/GLMDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/GLMDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.GLMTest;
+import org.apache.sysml.test.integration.applications.GLMTest;
@RunWith(value = Parameterized.class)
public class GLMDMLTest extends GLMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/GNMFDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/GNMFDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/GNMFDMLTest.java
index ceab76c..0ec1d62 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/GNMFDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/GNMFDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.GNMFTest;
+import org.apache.sysml.test.integration.applications.GNMFTest;
@RunWith(value = Parameterized.class)
public class GNMFDMLTest extends GNMFTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/HITSDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/HITSDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/HITSDMLTest.java
index 0b3f819..ad82867 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/HITSDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/HITSDMLTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.applications.HITSTest;
+import org.apache.sysml.test.integration.applications.HITSTest;
public class HITSDMLTest extends HITSTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/ID3DMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ID3DMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ID3DMLTest.java
index b46275c..7f7471f 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/ID3DMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ID3DMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ID3Test;
+import org.apache.sysml.test.integration.applications.ID3Test;
@RunWith(value = Parameterized.class)
public class ID3DMLTest extends ID3Test {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/L2SVMDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/L2SVMDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/L2SVMDMLTest.java
index 0b0eb19..df26e72 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/L2SVMDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/L2SVMDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.L2SVMTest;
+import org.apache.sysml.test.integration.applications.L2SVMTest;
@RunWith(value = Parameterized.class)
public class L2SVMDMLTest extends L2SVMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearLogRegDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearLogRegDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearLogRegDMLTest.java
index 3d8d811..522a34b 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearLogRegDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearLogRegDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.LinearLogRegTest;
+import org.apache.sysml.test.integration.applications.LinearLogRegTest;
@RunWith(value = Parameterized.class)
public class LinearLogRegDMLTest extends LinearLogRegTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearRegressionDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearRegressionDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearRegressionDMLTest.java
index e7f56a6..1901920 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearRegressionDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/LinearRegressionDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.LinearRegressionTest;
+import org.apache.sysml.test.integration.applications.LinearRegressionTest;
@RunWith(value = Parameterized.class)
public class LinearRegressionDMLTest extends LinearRegressionTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/MDABivariateStatsDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/MDABivariateStatsDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/MDABivariateStatsDMLTest.java
index fff5468..1e27ef9 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/MDABivariateStatsDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/MDABivariateStatsDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.MDABivariateStatsTest;
+import org.apache.sysml.test.integration.applications.MDABivariateStatsTest;
@RunWith(value = Parameterized.class)
public class MDABivariateStatsDMLTest extends MDABivariateStatsTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/MultiClassSVMDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/MultiClassSVMDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/MultiClassSVMDMLTest.java
index 16d8d75..68a53f9 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/MultiClassSVMDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/MultiClassSVMDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.MultiClassSVMTest;
+import org.apache.sysml.test.integration.applications.MultiClassSVMTest;
@RunWith(value = Parameterized.class)
public class MultiClassSVMDMLTest extends MultiClassSVMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/NaiveBayesDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/NaiveBayesDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/NaiveBayesDMLTest.java
index 027ad97..9d844a7 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/NaiveBayesDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/NaiveBayesDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.NaiveBayesTest;
+import org.apache.sysml.test.integration.applications.NaiveBayesTest;
@RunWith(value = Parameterized.class)
public class NaiveBayesDMLTest extends NaiveBayesTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/PageRankDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/PageRankDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/PageRankDMLTest.java
index 82a98a3..8fdf99e 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/PageRankDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/PageRankDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.PageRankTest;
+import org.apache.sysml.test.integration.applications.PageRankTest;
@RunWith(value = Parameterized.class)
public class PageRankDMLTest extends PageRankTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/dml/WelchTDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/WelchTDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/WelchTDMLTest.java
index b703df3..d0df8e1 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/dml/WelchTDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/WelchTDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.dml;
+package org.apache.sysml.test.integration.applications.dml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.WelchTTest;
+import org.apache.sysml.test.integration.applications.WelchTTest;
@RunWith(value = Parameterized.class)
public class WelchTDMLTest extends WelchTTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java
index 7fed05c..4e6e275 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForBivariateStatsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java
index b2b0adf..abdc0ac 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForCVMulticlassSVMTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java
index 034f78b..6d8c26a 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForCorrelationTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java
index b3f7294..cca5df0 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* Intension is to test file-based result merge with regard to its integration
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java
index 097dbe2..4517fe9 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForUnivariateStatsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForUnivariateStatsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForUnivariateStatsTest.java
index ee7573b..d8a783c 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForUnivariateStatsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForUnivariateStatsTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.parfor;
+package org.apache.sysml.test.integration.applications.parfor;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class ParForUnivariateStatsTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ApplyTransformPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ApplyTransformPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ApplyTransformPyDMLTest.java
index ed88c69..8d7995e 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ApplyTransformPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ApplyTransformPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ApplyTransformTest;
+import org.apache.sysml.test.integration.applications.ApplyTransformTest;
@RunWith(value = Parameterized.class)
public class ApplyTransformPyDMLTest extends ApplyTransformTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ArimaPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ArimaPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ArimaPyDMLTest.java
index 1c6ea48..b927c8a 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ArimaPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ArimaPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ArimaTest;
+import org.apache.sysml.test.integration.applications.ArimaTest;
@RunWith(value = Parameterized.class)
public class ArimaPyDMLTest extends ArimaTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineCGPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineCGPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineCGPyDMLTest.java
index bfa22c1..ccd568c 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineCGPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineCGPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.CsplineCGTest;
+import org.apache.sysml.test.integration.applications.CsplineCGTest;
@RunWith(value = Parameterized.class)
public class CsplineCGPyDMLTest extends CsplineCGTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineDSPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineDSPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineDSPyDMLTest.java
index b7403c1..dcb6e54 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineDSPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/CsplineDSPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.CsplineDSTest;
+import org.apache.sysml.test.integration.applications.CsplineDSTest;
@RunWith(value = Parameterized.class)
public class CsplineDSPyDMLTest extends CsplineDSTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GLMPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/GLMPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GLMPyDMLTest.java
index 275df76..d37cf52 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/GLMPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GLMPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.GLMTest;
+import org.apache.sysml.test.integration.applications.GLMTest;
@RunWith(value = Parameterized.class)
public class GLMPyDMLTest extends GLMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GNMFPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/GNMFPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GNMFPyDMLTest.java
index c8c1665..10b380a 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/GNMFPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/GNMFPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.GNMFTest;
+import org.apache.sysml.test.integration.applications.GNMFTest;
@RunWith(value = Parameterized.class)
public class GNMFPyDMLTest extends GNMFTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/HITSPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/HITSPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/HITSPyDMLTest.java
index 1675e9a..b80dbb5 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/HITSPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/HITSPyDMLTest.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.applications.HITSTest;
+import org.apache.sysml.test.integration.applications.HITSTest;
public class HITSPyDMLTest extends HITSTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ID3PyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ID3PyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ID3PyDMLTest.java
index 6eed0cb..8db9744 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/ID3PyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/ID3PyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.ID3Test;
+import org.apache.sysml.test.integration.applications.ID3Test;
@RunWith(value = Parameterized.class)
public class ID3PyDMLTest extends ID3Test {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/L2SVMPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/L2SVMPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/L2SVMPyDMLTest.java
index 91c0954..74e309e 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/L2SVMPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/L2SVMPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.L2SVMTest;
+import org.apache.sysml.test.integration.applications.L2SVMTest;
@RunWith(value = Parameterized.class)
public class L2SVMPyDMLTest extends L2SVMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearLogRegPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearLogRegPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearLogRegPyDMLTest.java
index 456ef52..a0e4015 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearLogRegPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearLogRegPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.LinearLogRegTest;
+import org.apache.sysml.test.integration.applications.LinearLogRegTest;
@RunWith(value = Parameterized.class)
public class LinearLogRegPyDMLTest extends LinearLogRegTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearRegressionPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearRegressionPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearRegressionPyDMLTest.java
index 67ebe5c..d830497 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearRegressionPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/LinearRegressionPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.LinearRegressionTest;
+import org.apache.sysml.test.integration.applications.LinearRegressionTest;
@RunWith(value = Parameterized.class)
public class LinearRegressionPyDMLTest extends LinearRegressionTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MDABivariateStatsPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/MDABivariateStatsPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MDABivariateStatsPyDMLTest.java
index 881de3f..e9ce5dd 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/MDABivariateStatsPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MDABivariateStatsPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.MDABivariateStatsTest;
+import org.apache.sysml.test.integration.applications.MDABivariateStatsTest;
@RunWith(value = Parameterized.class)
public class MDABivariateStatsPyDMLTest extends MDABivariateStatsTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MultiClassSVMPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/MultiClassSVMPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MultiClassSVMPyDMLTest.java
index acca074..b32d650 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/MultiClassSVMPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/MultiClassSVMPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.MultiClassSVMTest;
+import org.apache.sysml.test.integration.applications.MultiClassSVMTest;
@RunWith(value = Parameterized.class)
public class MultiClassSVMPyDMLTest extends MultiClassSVMTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/NaiveBayesPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/NaiveBayesPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/NaiveBayesPyDMLTest.java
index 83ca0be..c4301e0 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/NaiveBayesPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/NaiveBayesPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.NaiveBayesTest;
+import org.apache.sysml.test.integration.applications.NaiveBayesTest;
@RunWith(value = Parameterized.class)
public class NaiveBayesPyDMLTest extends NaiveBayesTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/PageRankPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/PageRankPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/PageRankPyDMLTest.java
index a4540f9..34f30a6 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/PageRankPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/PageRankPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.PageRankTest;
+import org.apache.sysml.test.integration.applications.PageRankTest;
@RunWith(value = Parameterized.class)
public class PageRankPyDMLTest extends PageRankTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/applications/pydml/WelchTPyDMLTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/pydml/WelchTPyDMLTest.java b/src/test/java/org/apache/sysml/test/integration/applications/pydml/WelchTPyDMLTest.java
index 77f7879..2b7f615 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/pydml/WelchTPyDMLTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/pydml/WelchTPyDMLTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.applications.pydml;
+package org.apache.sysml.test.integration.applications.pydml;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.ibm.bi.dml.test.integration.applications.WelchTTest;
+import org.apache.sysml.test.integration.applications.WelchTTest;
@RunWith(value = Parameterized.class)
public class WelchTPyDMLTest extends WelchTTest {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/AggregateInfTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/AggregateInfTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/AggregateInfTest.java
index 36e9f5d..28ee588 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/AggregateInfTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/AggregateInfTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumTest.java
index 2d68ec7..022d2cd 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumsSqTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumsSqTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumsSqTest.java
index b9ea24d..8d2d653 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumsSqTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColSumsSqTest.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullAggregateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullAggregateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullAggregateTest.java
index cd4ddcb..7ae3ba8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullAggregateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullAggregateTest.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullColAggregateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullColAggregateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullColAggregateTest.java
index af5c168..6a4c6d3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullColAggregateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullColAggregateTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* NOTES:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateTest.java
index d0b881d..87664b6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateTest.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import java.io.IOException;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullRowAggregateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullRowAggregateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullRowAggregateTest.java
index 2b753b2..1fe8af9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullRowAggregateTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullRowAggregateTest.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.aggregate;
+package org.apache.sysml.test.integration.functions.aggregate;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
* NOTES:
[67/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java
index 43f10a8..1675330 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.AppendM.CacheType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.AppendM.CacheType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AppendMInstruction extends AppendInstruction implements IDistributedCacheConsumer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java
index 4ed43a7..c09f8e5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AppendRInstruction extends AppendInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinUaggChainInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinUaggChainInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinUaggChainInstruction.java
index 7470a90..940f339 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinUaggChainInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinUaggChainInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryInstruction.java
index 87b0853..db18189 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class BinaryInstruction extends BinaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMInstruction.java
index 065abee..c87469b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMInstruction.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.AppendM.CacheType;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.AppendM.CacheType;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class BinaryMInstruction extends BinaryMRInstructionBase implements IDistributedCacheConsumer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMRInstructionBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMRInstructionBase.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMRInstructionBase.java
index c6beb15..664380a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMRInstructionBase.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/BinaryMRInstructionBase.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class BinaryMRInstructionBase extends MRInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CM_N_COVInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CM_N_COVInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CM_N_COVInstruction.java
index 315ee99..f7b8530 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CM_N_COVInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CM_N_COVInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
public class CM_N_COVInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVReblockInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVReblockInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVReblockInstruction.java
index 9a9ad03..e65894f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVReblockInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVReblockInstruction.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CSVReblockInstruction extends ReblockInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVWriteInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVWriteInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVWriteInstruction.java
index 3679891..86ce0ff 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVWriteInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CSVWriteInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CSVWriteInstruction extends UnaryMRInstructionBase{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineBinaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineBinaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineBinaryInstruction.java
index bd5a231..395b9e2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineBinaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineBinaryInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CombineBinaryInstruction extends BinaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineTernaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineTernaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineTernaryInstruction.java
index 6d36432..c9d06d3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineTernaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineTernaryInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.lops.Ternary.OperationTypes;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.lops.Ternary.OperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
public class CombineTernaryInstruction extends TernaryInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineUnaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineUnaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineUnaryInstruction.java
index b2280d1..1757712 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineUnaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CombineUnaryInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CombineUnaryInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeAggregateInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeAggregateInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeAggregateInstruction.java
index ae8306e..ab74540 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeAggregateInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeAggregateInstruction.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CumulativeAggregateInstruction extends AggregateUnaryInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java
index 504a92e..cc4da21 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public class CumulativeOffsetInstruction extends BinaryInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeSplitInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeSplitInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeSplitInstruction.java
index 5f08240..81162f0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeSplitInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeSplitInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataGenMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/DataGenMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataGenMRInstruction.java
index 445b0c8..6f86cb1 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/DataGenMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataGenMRInstruction.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class DataGenMRInstruction extends MRInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataPartitionMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/DataPartitionMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataPartitionMRInstruction.java
index cfc757c..c638a48 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/DataPartitionMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/DataPartitionMRInstruction.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
* Dummy instruction for cost estimation of data partition mr.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/GroupedAggregateInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/GroupedAggregateInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/GroupedAggregateInstruction.java
index 60338dc..9d940ae 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/GroupedAggregateInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/GroupedAggregateInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
public class GroupedAggregateInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/IDistributedCacheConsumer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/IDistributedCacheConsumer.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/IDistributedCacheConsumer.java
index 4d809e9..b39df08 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/IDistributedCacheConsumer.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/IDistributedCacheConsumer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/MMTSJMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MMTSJMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MMTSJMRInstruction.java
index 3204d25..d48efa5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MMTSJMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MMTSJMRInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
index e68422b..70a6cd5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class MRInstruction extends Instruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
index d778581..9fc7a16 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java
index 159b31f..2b0218c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/PMMJMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/PMMJMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/PMMJMRInstruction.java
index 849bd28..a8e9bf4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/PMMJMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/PMMJMRInstruction.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.PMMJ.CacheType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.PMMJ.CacheType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ParameterizedBuiltinMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ParameterizedBuiltinMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ParameterizedBuiltinMRInstruction.java
index 0bd2a83..6cbd849 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ParameterizedBuiltinMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ParameterizedBuiltinMRInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/PickByCountInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/PickByCountInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/PickByCountInstruction.java
index f05fab2..30e6eeb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/PickByCountInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/PickByCountInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.lops.PickByCount.OperationTypes;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.PickByCount.OperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class PickByCountInstruction extends MRInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
index 087f554..0f84600 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
@@ -15,36 +15,36 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/RandInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RandInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RandInstruction.java
index 4253435..9a31aac 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RandInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RandInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class RandInstruction extends DataGenMRInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
index 4976138..2edab99 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReIndexOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReIndexOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReblockInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReblockInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReblockInstruction.java
index a09c77f..4055af3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReblockInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReblockInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class ReblockInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/RemoveEmptyMRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RemoveEmptyMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RemoveEmptyMRInstruction.java
index a5a6c65..d3b7f8c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RemoveEmptyMRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RemoveEmptyMRInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReorgInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReorgInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReorgInstruction.java
index 8a21e06..c5f071a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReorgInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReorgInstruction.java
@@ -15,23 +15,23 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class ReorgInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReplicateInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReplicateInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReplicateInstruction.java
index c13e691..a42e042 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ReplicateInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ReplicateInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ScalarInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ScalarInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ScalarInstruction.java
index f47dfe9..0f91a25 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ScalarInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ScalarInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
public class ScalarInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/SeqInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/SeqInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/SeqInstruction.java
index 0a5e679..197bde8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/SeqInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/SeqInstruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class SeqInstruction extends DataGenMRInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/TernaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/TernaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/TernaryInstruction.java
index 8e19dc2..bd03278 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/TernaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/TernaryInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.Ternary;
-import com.ibm.bi.dml.lops.Ternary.OperationTypes;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.lops.Ternary;
+import org.apache.sysml.lops.Ternary.OperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
public class TernaryInstruction extends MRInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/UaggOuterChainInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/UaggOuterChainInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/UaggOuterChainInstruction.java
index a414415..c888d5b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/UaggOuterChainInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/UaggOuterChainInstruction.java
@@ -15,32 +15,32 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
import java.util.Arrays;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixOuterAgg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixOuterAgg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryInstruction.java
index 194c8e8..9b9dd66 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public class UnaryInstruction extends UnaryMRInstructionBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryMRInstructionBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryMRInstructionBase.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryMRInstructionBase.java
index a869e93..6ce1da4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryMRInstructionBase.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/UnaryMRInstructionBase.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class UnaryMRInstructionBase extends MRInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java
index fc8d3da..8f8eeaf 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.mr;
+package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.CachedValueMap;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ZeroOutOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ZeroOutOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* ZeroOut with complementary=false is to zero out a subregion inside a matrix
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateTernarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateTernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateTernarySPInstruction.java
index 7c7806f..7f13b2b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateTernarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateTernarySPInstruction.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateUnarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateUnarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateUnarySPInstruction.java
index ae38aeb..87eb4ba 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateUnarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AggregateUnarySPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
@@ -23,27 +23,27 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.AggregateDropCorrectionFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.FilterDiagBlocksFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.functions.AggregateDropCorrectionFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.FilterDiagBlocksFunction;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
/**
*
[62/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/RuntimeDataFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/RuntimeDataFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/data/RuntimeDataFormat.java
index 928ba99..dc0014f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/RuntimeDataFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/RuntimeDataFormat.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public enum RuntimeDataFormat
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
index 404b64b..71a1bd1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
@@ -16,12 +16,12 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.Serializable;
import java.util.Arrays;
-import com.ibm.bi.dml.runtime.util.SortUtils;
+import org.apache.sysml.runtime.util.SortUtils;
public class SparseRow implements Serializable
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowsIterator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowsIterator.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowsIterator.java
index 070af86..f3ddb0c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowsIterator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowsIterator.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java b/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java
index ab22ed7..a4ac227 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedAdaptivePartialBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedAdaptivePartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedAdaptivePartialBlock.java
index 2485d37..194d7f2 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedAdaptivePartialBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedAdaptivePartialBlock.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedAdaptivePartialBlock extends Tagged<AdaptivePartialBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java
index 0e07fa5..14102e3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -31,9 +31,9 @@ import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.UtilFunctions;
//sorted by first index, tag, and second index
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
index e121713..9ad0806 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.IntWritable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixBlock.java
index e57f344..76817b5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixBlock.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedMatrixBlock extends TaggedMatrixValue
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixCell.java
index dcd4113..7eb70db 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixCell.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedMatrixCell extends TaggedMatrixValue
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixPackedCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixPackedCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixPackedCell.java
index 3ec86d7..7b46bf9 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixPackedCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixPackedCell.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedMatrixPackedCell extends TaggedMatrixValue
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
index 019647d..87a1eda 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedMatrixValue extends Tagged<MatrixValue>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
index 5e1947d..e300d4a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class TaggedPartialBlock extends Tagged<PartialBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java
index 616438f..6453cee 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -27,7 +27,7 @@ import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java
index 133c94a..acd75d1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java
@@ -16,12 +16,12 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
public class TextCellToRowBlockConverter implements Converter<LongWritable, Text, MatrixIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java
index 367896f..fbb1ee4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java
@@ -16,12 +16,12 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
public class TextToBinaryCellConverter
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java
index b068eae..4427e61 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class TripleIndexes implements WritableComparable<TripleIndexes>, Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java
index 5f55cf2..c6cdb04 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java
index 2d958c2..3b826d5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -24,12 +24,12 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Map;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public class WeightedCell extends MatrixCell
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java
index 47a389e..c6d6dd7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java
index 088041f..9443308 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java
@@ -16,13 +16,13 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
public class WeightedPair extends WeightedCell
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
index 4508ad7..7376280 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/DelegatingInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/DelegatingInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/DelegatingInputFormat.java
index bb3a344..df93910 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/DelegatingInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/DelegatingInputFormat.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package com.ibm.bi.dml.runtime.matrix.data.hadoopfix;
+package org.apache.sysml.runtime.matrix.data.hadoopfix;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
index 50724ac3..5ef7f3d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.ibm.bi.dml.runtime.matrix.data.hadoopfix;
+package org.apache.sysml.runtime.matrix.data.hadoopfix;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/TaggedInputSplit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/TaggedInputSplit.java b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/TaggedInputSplit.java
index 2e08c42..4f7ad58 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/TaggedInputSplit.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/TaggedInputSplit.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package com.ibm.bi.dml.runtime.matrix.data.hadoopfix;
+package org.apache.sysml.runtime.matrix.data.hadoopfix;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java
index 6f9ed25..3adca0c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.HashMap;
@@ -27,15 +27,15 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.CM_N_COVCell;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedPair;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.matrix.data.CM_N_COVCell;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.data.WeightedPair;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
public class CMCOVMRMapper extends MapperBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java
index adcab8f..794ee3d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,19 +29,19 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.CM_N_COVCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.matrix.data.CM_N_COVCell;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
public class CMCOVMRReducer extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDMapper.java
index 14e19da..3e9c29e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.regex.Pattern;
@@ -32,12 +32,12 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.wink.json4j.JSONException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.transform.TfUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.transform.TfUtils;
public class CSVAssignRowIDMapper extends MapReduceBase implements Mapper<LongWritable, Text, ByteWritable, OffsetCount>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDReducer.java
index 8f29199..952ab4c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVAssignRowIDReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,8 +28,8 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
public class CSVAssignRowIDReducer extends MapReduceBase implements Reducer<ByteWritable, OffsetCount, ByteWritable, OffsetCount>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
index c0e1090..261cd46 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,14 +33,14 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.BlockRow;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.OffsetCount;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.CSVReblockMR;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.BlockRow;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.OffsetCount;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class CSVReblockMapper extends MapperBase implements Mapper<LongWritable, Text, TaggedFirstSecondIndexes, BlockRow>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockReducer.java
index ec02541..6b63e31 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.HashMap;
@@ -26,15 +26,15 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR.BlockRow;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.matrix.CSVReblockMR.BlockRow;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
public class CSVReblockReducer extends ReduceBase implements Reducer<TaggedFirstSecondIndexes, BlockRow, MatrixIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java
index eea2cd9..d42069e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,11 +27,11 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
public class CSVWriteMapper extends MapperBase implements Mapper<Writable, Writable, TaggedFirstSecondIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
index f1e0f0b..0a8ca90 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.DataInput;
import java.io.DataOutput;
@@ -32,15 +32,15 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput.Situation;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput;
+import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput.Situation;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class CSVWriteReducer extends ReduceBase implements Reducer<TaggedFirstSecondIndexes, MatrixBlock, NullWritable, RowBlockForTextOutput>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
index 2bcb81d..eeef630 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.util.ArrayList;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
index 644b5fd..f3a873c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
public abstract class CachedMapElement
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java
index c9b975c..c1d4e1d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java
@@ -16,14 +16,14 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
public class CachedValueMap //extends CachedMap<IndexedMatrixValue>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java
index e731d69..d8b2c1a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
@@ -24,9 +24,9 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.MultipleOutputs;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.Pair;
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
index 21b7c86..22256b6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
@@ -26,14 +26,14 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.matrix.data.LibMatrixDatagen;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.RandomMatrixGenerator;
public class DataGenMapper extends GMRMapper
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java
index a8a4da7..e26fc30 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.File;
import java.util.ArrayList;
import org.apache.hadoop.fs.Path;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.io.ReaderBinaryBlock;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.io.ReaderBinaryBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.DataConverter;
public class DistributedCacheInput
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java
index e958f56..16fa7f1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.Iterator;
@@ -26,10 +26,10 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixPackedCell;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixPackedCell;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
public class GMRCombiner extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCtableBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCtableBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCtableBuffer.java
index ec31b61..a303bf7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCtableBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCtableBuffer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.util.ArrayList;
import java.util.HashMap;
@@ -24,13 +24,13 @@ import java.util.Map.Entry;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.MatrixWriter;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
public class GMRCtableBuffer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java
index 78e6b29..f529c7c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,12 +28,12 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixPackedCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixPackedCell;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixPackedCell;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixPackedCell;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRReducer.java
index c0bbf60..93f1057 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,15 +27,15 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixPackedCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.instructions.mr.AppendRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.TernaryInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixPackedCell;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
public class GMRReducer extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRCombiner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRCombiner.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRCombiner.java
index e99d381..db9ad65 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRCombiner.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRCombiner.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.HashMap;
@@ -26,16 +26,16 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedInt;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.matrix.data.TaggedInt;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class GroupedAggMRCombiner extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java
index feea373..0d4df46 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,12 +28,12 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedInt;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedInt;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
public class GroupedAggMRMapper extends MapperBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRReducer.java
index 061c0a5..dbaccf5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,17 +28,17 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedInt;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TaggedInt;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class GroupedAggMRReducer extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java
index e09002d..ea1d08f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java
@@ -16,12 +16,12 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
public class IndexedMatrixValue implements Serializable
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java
index bf53bbe..feb043f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.HashMap;
@@ -25,11 +25,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* Base class for MMCJ partial aggregator and MMCJ input cache to factor out
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
index 8848bd2..aba91ef 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.Iterator;
@@ -26,8 +26,8 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
public class MMCJMRCombiner extends MMCJMRCombinerReducerBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombinerReducerBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombinerReducerBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombinerReducerBase.java
index 2aa38ae..1b990ce 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombinerReducerBase.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombinerReducerBase.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,14 +24,14 @@ import java.util.Iterator;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
public class MMCJMRCombinerReducerBase extends ReduceBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
index 473dc96..875e27f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.Pair;
public class MMCJMRInputCache extends MMCJMRCache
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java
index 4119467..ba31415 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,12 +27,12 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
public class MMCJMRMapper extends MapperBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
index 05165e2..bdf4964 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,15 +30,15 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class MMCJMRReducer extends MMCJMRCombinerReducerBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducerWithAggregator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducerWithAggregator.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducerWithAggregator.java
index fe10ded..15a3a8b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducerWithAggregator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducerWithAggregator.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.Iterator;
@@ -27,16 +27,16 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.MMCJ.MMCJType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.MMCJ.MMCJType;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class MMCJMRReducerWithAggregator extends MMCJMRCombinerReducerBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java
index da5104d..14c0d7f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,12 +29,12 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TripleIndexes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.matrix.data.TripleIndexes;
public class MMRJMRMapper extends MapperBase
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java
index dc290ec..8b5c39c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.Iterator;
@@ -26,15 +26,15 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.TripleIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
+import org.apache.sysml.runtime.matrix.data.TripleIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRBaseForCommonInstructions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRBaseForCommonInstructions.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRBaseForCommonInstructions.java
index a6aedac..9d2f20d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRBaseForCommonInstructions.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRBaseForCommonInstructions.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,29 +28,29 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateUnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendGInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeSplitInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MatrixReshapeMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RangeBasedReIndexInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReorgInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.ZeroOutInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateUnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendGInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendMInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.CumulativeAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.CumulativeSplitInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MatrixReshapeMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.RangeBasedReIndexInstruction;
+import org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReorgInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.ZeroOutInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRConfigurationNames.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRConfigurationNames.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRConfigurationNames.java
index 578abae..b49e67b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRConfigurationNames.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRConfigurationNames.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
index 1e21292..8100972 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.mapred;
+package org.apache.sysml.runtime.matrix.mapred;
import java.io.IOException;
import java.util.ArrayList;
@@ -41,59 +41,59 @@ import org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat;
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendGInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MapMultChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PMMJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.io.BinaryBlockSerialization;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.AddDummyWeightConverter;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryBlockToBinaryCellConverter;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryBlockToRowBlockConverter;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryBlockToTextCellConverter;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryCellToRowBlockConverter;
-import com.ibm.bi.dml.runtime.matrix.data.BinaryCellToTextConverter;
-import com.ibm.bi.dml.runtime.matrix.data.Converter;
-import com.ibm.bi.dml.runtime.matrix.data.IdenticalConverter;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue;
-import com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TextCellToRowBlockConverter;
-import com.ibm.bi.dml.runtime.matrix.data.TextToBinaryCellConverter;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCellToSortInputConverter;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedPair;
-import com.ibm.bi.dml.runtime.matrix.data.hadoopfix.MultipleInputs;
-import com.ibm.bi.dml.runtime.matrix.sort.SamplingSortMRInputFormat;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendGInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendMInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMInstruction;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MapMultChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryMRInstructionBase;
+import org.apache.sysml.runtime.io.BinaryBlockSerialization;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.AddDummyWeightConverter;
+import org.apache.sysml.runtime.matrix.data.BinaryBlockToBinaryCellConverter;
+import org.apache.sysml.runtime.matrix.data.BinaryBlockToRowBlockConverter;
+import org.apache.sysml.runtime.matrix.data.BinaryBlockToTextCellConverter;
+import org.apache.sysml.runtime.matrix.data.BinaryCellToRowBlockConverter;
+import org.apache.sysml.runtime.matrix.data.BinaryCellToTextConverter;
+import org.apache.sysml.runtime.matrix.data.Converter;
+import org.apache.sysml.runtime.matrix.data.IdenticalConverter;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue;
+import org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TextCellToRowBlockConverter;
+import org.apache.sysml.runtime.matrix.data.TextToBinaryCellConverter;
+import org.apache.sysml.runtime.matrix.data.WeightedCellToSortInputConverter;
+import org.apache.sysml.runtime.matrix.data.WeightedPair;
+import org.apache.sysml.runtime.matrix.data.hadoopfix.MultipleInputs;
+import org.apache.sysml.runtime.matrix.sort.SamplingSortMRInputFormat;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
@SuppressWarnings({ "rawtypes", "deprecation" })
public class MRJobConfiguration
[48/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/MLOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/MLOutput.java b/src/main/java/com/ibm/bi/dml/api/MLOutput.java
deleted file mode 100644
index 82f74a0..0000000
--- a/src/main/java/com/ibm/bi/dml/api/MLOutput.java
+++ /dev/null
@@ -1,401 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-import org.apache.spark.mllib.linalg.DenseVector;
-import org.apache.spark.mllib.linalg.VectorUDT;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.StructField;
-import org.apache.spark.sql.types.StructType;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMLBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-/**
- * This is a simple container object that returns the output of execute from MLContext
- *
- */
-public class MLOutput {
-
-
-
- HashMap<String, JavaPairRDD<MatrixIndexes,MatrixBlock>> _outputs;
- private HashMap<String, MatrixCharacteristics> _outMetadata = null;
-
- public MLOutput(HashMap<String, JavaPairRDD<MatrixIndexes,MatrixBlock>> outputs, HashMap<String, MatrixCharacteristics> outMetadata) {
- this._outputs = outputs;
- this._outMetadata = outMetadata;
- }
-
- public JavaPairRDD<MatrixIndexes,MatrixBlock> getBinaryBlockedRDD(String varName) throws DMLRuntimeException {
- if(_outputs.containsKey(varName)) {
- return _outputs.get(varName);
- }
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
-
- public MatrixCharacteristics getMatrixCharacteristics(String varName) throws DMLRuntimeException {
- if(_outputs.containsKey(varName)) {
- return _outMetadata.get(varName);
- }
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
-
- /**
- * Note, the output DataFrame has an additional column ID.
- * An easy way to get DataFrame without ID is by df.sort("ID").drop("ID")
- * @param sqlContext
- * @param varName
- * @return
- * @throws DMLRuntimeException
- */
- public DataFrame getDF(SQLContext sqlContext, String varName) throws DMLRuntimeException {
- JavaPairRDD<MatrixIndexes,MatrixBlock> rdd = getBinaryBlockedRDD(varName);
- if(rdd != null) {
- MatrixCharacteristics mc = _outMetadata.get(varName);
- return RDDConverterUtilsExt.binaryBlockToDataFrame(rdd, mc, sqlContext);
- }
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
-
- /**
- *
- * @param sqlContext
- * @param varName
- * @param outputVector if true, returns DataFrame with two column: ID and org.apache.spark.mllib.linalg.Vector
- * @return
- * @throws DMLRuntimeException
- */
- public DataFrame getDF(SQLContext sqlContext, String varName, boolean outputVector) throws DMLRuntimeException {
- if(outputVector) {
- JavaPairRDD<MatrixIndexes,MatrixBlock> rdd = getBinaryBlockedRDD(varName);
- if(rdd != null) {
- MatrixCharacteristics mc = _outMetadata.get(varName);
- return RDDConverterUtilsExt.binaryBlockToVectorDataFrame(rdd, mc, sqlContext);
- }
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
- else {
- return getDF(sqlContext, varName);
- }
-
- }
-
- /**
- * This methods improves the performance of MLPipeline wrappers.
- * @param sqlContext
- * @param varName
- * @param range range is inclusive
- * @return
- * @throws DMLRuntimeException
- */
- public DataFrame getDF(SQLContext sqlContext, String varName, HashMap<String, Tuple2<Long, Long>> range) throws DMLRuntimeException {
- JavaPairRDD<MatrixIndexes,MatrixBlock> binaryBlockRDD = getBinaryBlockedRDD(varName);
- if(binaryBlockRDD == null) {
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
- MatrixCharacteristics mc = _outMetadata.get(varName);
- long rlen = mc.getRows(); long clen = mc.getCols();
- int brlen = mc.getRowsPerBlock(); int bclen = mc.getColsPerBlock();
-
- ArrayList<Tuple2<String, Tuple2<Long, Long>>> alRange = new ArrayList<Tuple2<String, Tuple2<Long, Long>>>();
- for(Entry<String, Tuple2<Long, Long>> e : range.entrySet()) {
- alRange.add(new Tuple2<String, Tuple2<Long,Long>>(e.getKey(), e.getValue()));
- }
-
- // Very expensive operation here: groupByKey (where number of keys might be too large)
- JavaRDD<Row> rowsRDD = binaryBlockRDD.flatMapToPair(new ProjectRows(rlen, clen, brlen, bclen))
- .groupByKey().map(new ConvertDoubleArrayToRangeRows(clen, bclen, alRange));
-
- int numColumns = (int) clen;
- if(numColumns <= 0) {
- throw new DMLRuntimeException("Output dimensions unknown after executing the script and hence cannot create the dataframe");
- }
-
- List<StructField> fields = new ArrayList<StructField>();
- // LongTypes throw an error: java.lang.Double incompatible with java.lang.Long
- fields.add(DataTypes.createStructField("ID", DataTypes.DoubleType, false));
- for(int k = 0; k < alRange.size(); k++) {
- String colName = alRange.get(k)._1;
- long low = alRange.get(k)._2._1;
- long high = alRange.get(k)._2._2;
- if(low != high)
- fields.add(DataTypes.createStructField(colName, new VectorUDT(), false));
- else
- fields.add(DataTypes.createStructField(colName, DataTypes.DoubleType, false));
- }
-
- // This will cause infinite recursion due to bug in Spark
- // https://issues.apache.org/jira/browse/SPARK-6999
- // return sqlContext.createDataFrame(rowsRDD, colNames); // where ArrayList<String> colNames
- return sqlContext.createDataFrame(rowsRDD.rdd(), DataTypes.createStructType(fields));
-
- }
-
- public JavaRDD<String> getStringRDD(String varName, String format) throws DMLRuntimeException {
- if(format.compareTo("text") == 0) {
- JavaPairRDD<MatrixIndexes, MatrixBlock> binaryRDD = getBinaryBlockedRDD(varName);
- MatrixCharacteristics mcIn = getMatrixCharacteristics(varName);
- return RDDConverterUtilsExt.binaryBlockToStringRDD(binaryRDD, mcIn, format);
- }
-// else if(format.compareTo("csv") == 0) {
-//
-// }
- else {
- throw new DMLRuntimeException("The output format:" + format + " is not implemented yet.");
- }
-
- }
-
- public MLMatrix getMLMatrix(MLContext ml, SQLContext sqlContext, String varName) throws DMLRuntimeException {
- JavaPairRDD<MatrixIndexes,MatrixBlock> rdd = getBinaryBlockedRDD(varName);
- if(rdd != null) {
- MatrixCharacteristics mc = getMatrixCharacteristics(varName);
- StructType schema = MLBlock.getDefaultSchemaForBinaryBlock();
- return new MLMatrix(sqlContext.createDataFrame(rdd.map(new GetMLBlock()).rdd(), schema), mc, ml);
- }
- throw new DMLRuntimeException("Variable " + varName + " not found in the output symbol table.");
- }
-
-// /**
-// * Experimental: Please use this with caution as it will fail in many corner cases.
-// * @return org.apache.spark.mllib.linalg.distributed.BlockMatrix
-// * @throws DMLRuntimeException
-// */
-// public BlockMatrix getMLLibBlockedMatrix(MLContext ml, SQLContext sqlContext, String varName) throws DMLRuntimeException {
-// return getMLMatrix(ml, sqlContext, varName).toBlockedMatrix();
-// }
-
- public static class ProjectRows implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, Long, Tuple2<Long, Double[]>> {
- private static final long serialVersionUID = -4792573268900472749L;
- long rlen; long clen;
- int brlen; int bclen;
- public ProjectRows(long rlen, long clen, int brlen, int bclen) {
- this.rlen = rlen;
- this.clen = clen;
- this.brlen = brlen;
- this.bclen = bclen;
- }
-
- @Override
- public Iterable<Tuple2<Long, Tuple2<Long, Double[]>>> call(Tuple2<MatrixIndexes, MatrixBlock> kv) throws Exception {
- // ------------------------------------------------------------------
- // Compute local block size:
- // Example: For matrix: 1500 X 1100 with block length 1000 X 1000
- // We will have four local block sizes (1000X1000, 1000X100, 500X1000 and 500X1000)
- long blockRowIndex = kv._1.getRowIndex();
- long blockColIndex = kv._1.getColumnIndex();
- int lrlen = UtilFunctions.computeBlockSize(rlen, blockRowIndex, brlen);
- int lclen = UtilFunctions.computeBlockSize(clen, blockColIndex, bclen);
- // ------------------------------------------------------------------
-
- long startRowIndex = (kv._1.getRowIndex()-1) * bclen;
- MatrixBlock blk = kv._2;
- ArrayList<Tuple2<Long, Tuple2<Long, Double[]>>> retVal = new ArrayList<Tuple2<Long,Tuple2<Long,Double[]>>>();
- for(int i = 0; i < lrlen; i++) {
- Double[] partialRow = new Double[lclen];
- for(int j = 0; j < lclen; j++) {
- partialRow[j] = blk.getValue(i, j);
- }
- retVal.add(new Tuple2<Long, Tuple2<Long,Double[]>>(startRowIndex + i, new Tuple2<Long,Double[]>(kv._1.getColumnIndex(), partialRow)));
- }
- return (Iterable<Tuple2<Long, Tuple2<Long, Double[]>>>) retVal;
- }
- }
-
- public static class ConvertDoubleArrayToRows implements Function<Tuple2<Long, Iterable<Tuple2<Long, Double[]>>>, Row> {
- private static final long serialVersionUID = 4441184411670316972L;
-
- int bclen; long clen;
- boolean outputVector;
- public ConvertDoubleArrayToRows(long clen, int bclen, boolean outputVector) {
- this.bclen = bclen;
- this.clen = clen;
- this.outputVector = outputVector;
- }
-
- @Override
- public Row call(Tuple2<Long, Iterable<Tuple2<Long, Double[]>>> arg0)
- throws Exception {
-
- HashMap<Long, Double[]> partialRows = new HashMap<Long, Double[]>();
- int sizeOfPartialRows = 0;
- for(Tuple2<Long, Double[]> kv : arg0._2) {
- partialRows.put(kv._1, kv._2);
- sizeOfPartialRows += kv._2.length;
- }
-
- // Insert first row as row index
- Object[] row = null;
- if(outputVector) {
- row = new Object[2];
- double [] vecVals = new double[sizeOfPartialRows];
-
- for(long columnBlockIndex = 1; columnBlockIndex <= partialRows.size(); columnBlockIndex++) {
- if(partialRows.containsKey(columnBlockIndex)) {
- Double [] array = partialRows.get(columnBlockIndex);
- // ------------------------------------------------------------------
- // Compute local block size:
- int lclen = UtilFunctions.computeBlockSize(clen, columnBlockIndex, bclen);
- // ------------------------------------------------------------------
- if(array.length != lclen) {
- throw new Exception("Incorrect double array provided by ProjectRows");
- }
- for(int i = 0; i < lclen; i++) {
- vecVals[(int) ((columnBlockIndex-1)*bclen + i)] = array[i];
- }
- }
- else {
- throw new Exception("The block for column index " + columnBlockIndex + " is missing. Make sure the last instruction is not returning empty blocks");
- }
- }
-
- long rowIndex = arg0._1;
- row[0] = new Double(rowIndex);
- row[1] = new DenseVector(vecVals); // breeze.util.JavaArrayOps.arrayDToDv(vecVals);
- }
- else {
- row = new Double[sizeOfPartialRows + 1];
- long rowIndex = arg0._1;
- row[0] = new Double(rowIndex);
- for(long columnBlockIndex = 1; columnBlockIndex <= partialRows.size(); columnBlockIndex++) {
- if(partialRows.containsKey(columnBlockIndex)) {
- Double [] array = partialRows.get(columnBlockIndex);
- // ------------------------------------------------------------------
- // Compute local block size:
- int lclen = UtilFunctions.computeBlockSize(clen, columnBlockIndex, bclen);
- // ------------------------------------------------------------------
- if(array.length != lclen) {
- throw new Exception("Incorrect double array provided by ProjectRows");
- }
- for(int i = 0; i < lclen; i++) {
- row[(int) ((columnBlockIndex-1)*bclen + i) + 1] = array[i];
- }
- }
- else {
- throw new Exception("The block for column index " + columnBlockIndex + " is missing. Make sure the last instruction is not returning empty blocks");
- }
- }
- }
- Object[] row_fields = row;
- return RowFactory.create(row_fields);
- }
- }
-
-
- public static class ConvertDoubleArrayToRangeRows implements Function<Tuple2<Long, Iterable<Tuple2<Long, Double[]>>>, Row> {
- private static final long serialVersionUID = 4441184411670316972L;
-
- int bclen; long clen;
- ArrayList<Tuple2<String, Tuple2<Long, Long>>> range;
- public ConvertDoubleArrayToRangeRows(long clen, int bclen, ArrayList<Tuple2<String, Tuple2<Long, Long>>> range) {
- this.bclen = bclen;
- this.clen = clen;
- this.range = range;
- }
-
- @Override
- public Row call(Tuple2<Long, Iterable<Tuple2<Long, Double[]>>> arg0)
- throws Exception {
-
- HashMap<Long, Double[]> partialRows = new HashMap<Long, Double[]>();
- int sizeOfPartialRows = 0;
- for(Tuple2<Long, Double[]> kv : arg0._2) {
- partialRows.put(kv._1, kv._2);
- sizeOfPartialRows += kv._2.length;
- }
-
- // Insert first row as row index
- Object[] row = null;
- row = new Object[range.size() + 1];
-
- double [] vecVals = new double[sizeOfPartialRows];
-
- for(long columnBlockIndex = 1; columnBlockIndex <= partialRows.size(); columnBlockIndex++) {
- if(partialRows.containsKey(columnBlockIndex)) {
- Double [] array = partialRows.get(columnBlockIndex);
- // ------------------------------------------------------------------
- // Compute local block size:
- int lclen = UtilFunctions.computeBlockSize(clen, columnBlockIndex, bclen);
- // ------------------------------------------------------------------
- if(array.length != lclen) {
- throw new Exception("Incorrect double array provided by ProjectRows");
- }
- for(int i = 0; i < lclen; i++) {
- vecVals[(int) ((columnBlockIndex-1)*bclen + i)] = array[i];
- }
- }
- else {
- throw new Exception("The block for column index " + columnBlockIndex + " is missing. Make sure the last instruction is not returning empty blocks");
- }
- }
-
- long rowIndex = arg0._1;
- row[0] = new Double(rowIndex);
-
- int i = 1;
-
- //for(Entry<String, Tuple2<Long, Long>> e : range.entrySet()) {
- for(int k = 0; k < range.size(); k++) {
- long low = range.get(k)._2._1;
- long high = range.get(k)._2._2;
-
- if(high < low) {
- throw new Exception("Incorrect range:" + high + "<" + low);
- }
-
- if(low == high) {
- row[i] = new Double(vecVals[(int) (low-1)]);
- }
- else {
- int lengthOfVector = (int) (high - low + 1);
- double [] tempVector = new double[lengthOfVector];
- for(int j = 0; j < lengthOfVector; j++) {
- tempVector[j] = vecVals[(int) (low + j - 1)];
- }
- row[i] = new DenseVector(tempVector);
- }
-
- i++;
- }
-
- Object[] row_fields = row;
- return RowFactory.create(row_fields);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/jmlc/Connection.java b/src/main/java/com/ibm/bi/dml/api/jmlc/Connection.java
deleted file mode 100644
index 57d389f..0000000
--- a/src/main/java/com/ibm/bi/dml/api/jmlc/Connection.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api.jmlc;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteRemovePersistentReadWrite;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.io.ReaderTextCell;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- * JMLC (Java Machine Learning Connector) API:
- *
- * NOTES:
- * * Currently fused API and implementation in order to reduce complexity.
- * * See SystemTMulticlassSVMScoreTest for an usage example.
- */
-public class Connection
-{
-
- private DMLConfig _conf = null;
-
- /**
- * Connection constructor, starting point for any other JMLC API calls.
- *
- */
- public Connection()
- {
- //setup basic parameters for embedded execution
- DataExpression.REJECT_READ_UNKNOWN_SIZE = false;
- DMLScript.rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = false;
- CacheableData.disableCaching();
-
- //create default configuration
- _conf = new DMLConfig();
- ConfigurationManager.setConfig(_conf);
- }
-
- /**
- *
- * @param script
- * @param inputs
- * @param outputs
- * @return
- * @throws DMLException
- */
- public PreparedScript prepareScript( String script, String[] inputs, String[] outputs, boolean parsePyDML)
- throws DMLException
- {
- return prepareScript(script, new HashMap<String,String>(), inputs, outputs, parsePyDML);
- }
-
- /**
- *
- * @param script
- * @param args
- * @param inputs
- * @param outputs
- * @return
- * @throws DMLException
- */
- public PreparedScript prepareScript( String script, HashMap<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML)
- throws DMLException
- {
- //prepare arguments
-
- //simplified compilation chain
- Program rtprog = null;
- try
- {
- //parsing
- AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
- DMLProgram prog = parser.parse(null, script, args);
-
- //language validate
- DMLTranslator dmlt = new DMLTranslator(prog);
- dmlt.liveVariableAnalysis(prog);
- dmlt.validateParseTree(prog);
-
- //hop construct/rewrite
- dmlt.constructHops(prog);
- dmlt.rewriteHopsDAG(prog);
-
- //rewrite persistent reads/writes
- RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs);
- ProgramRewriter rewriter2 = new ProgramRewriter(rewrite);
- rewriter2.rewriteProgramHopDAGs(prog);
-
- //lop construct and runtime prog generation
- dmlt.constructLops(prog);
- rtprog = prog.getRuntimeProgram(_conf);
-
- //final cleanup runtime prog
- JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
-
- //System.out.println(Explain.explain(rtprog));
- }
- catch(Exception ex)
- {
- throw new DMLException(ex);
- }
-
- //return newly create precompiled script
- return new PreparedScript(rtprog, inputs, outputs);
- }
-
- /**
- *
- */
- public void close()
- {
-
- }
-
- /**
- *
- * @param fname
- * @return
- * @throws IOException
- */
- public String readScript(String fname)
- throws IOException
- {
- StringBuilder sb = new StringBuilder();
- BufferedReader in = null;
- try
- {
- //read from hdfs or gpfs file system
- if( fname.startsWith("hdfs:")
- || fname.startsWith("gpfs:") )
- {
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path scriptPath = new Path(fname);
- in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
- }
- // from local file system
- else
- {
- in = new BufferedReader(new FileReader(fname));
- }
-
- //core script reading
- String tmp = null;
- while ((tmp = in.readLine()) != null)
- {
- sb.append( tmp );
- sb.append( "\n" );
- }
- }
- catch (IOException ex)
- {
- throw ex;
- }
- finally
- {
- if( in != null )
- in.close();
- }
-
- return sb.toString();
- }
-
- /**
- * Converts an input string representation of a matrix in textcell format
- * into a dense double array. The number of rows and columns need to be
- * specified because textcell only represents non-zero values and hence
- * does not define the dimensions in the general case.
- *
- * @param input a string representation of an input matrix,
- * in format textcell (rowindex colindex value)
- * @param rows number of rows
- * @param cols number of columns
- * @return
- * @throws IOException
- */
- public double[][] convertToDoubleMatrix(String input, int rows, int cols)
- throws IOException
- {
- double[][] ret = null;
-
- try
- {
- //read input matrix
- InputStream is = new ByteArrayInputStream(input.getBytes("UTF-8"));
- ReaderTextCell reader = (ReaderTextCell)MatrixReaderFactory.createMatrixReader(InputInfo.TextCellInputInfo);
- MatrixBlock mb = reader.readMatrixFromInputStream(is, rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long)rows*cols);
-
- //convert to double array
- ret = DataConverter.convertToDoubleMatrix( mb );
- }
- catch(DMLRuntimeException rex)
- {
- throw new IOException( rex );
- }
-
- return ret;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/jmlc/JMLCUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/jmlc/JMLCUtils.java b/src/main/java/com/ibm/bi/dml/api/jmlc/JMLCUtils.java
deleted file mode 100644
index 3cf3e18..0000000
--- a/src/main/java/com/ibm/bi/dml/api/jmlc/JMLCUtils.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api.jmlc;
-
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-
-public class JMLCUtils
-{
-
-
- /**
- * Removes rmvar instructions that would remove any of the given outputs.
- * This is important for keeping registered outputs after the program terminates.
- *
- * @param prog
- */
- public static void cleanupRuntimeProgram( Program prog, String[] outputs)
- {
- Map<String, FunctionProgramBlock> funcMap = prog.getFunctionProgramBlocks();
- if( funcMap != null && !funcMap.isEmpty() )
- {
- for( Entry<String, FunctionProgramBlock> e : funcMap.entrySet() )
- {
- FunctionProgramBlock fpb = e.getValue();
- for( ProgramBlock pb : fpb.getChildBlocks() )
- rCleanupRuntimeProgram(pb, outputs);
- }
- }
-
- for( ProgramBlock pb : prog.getProgramBlocks() )
- rCleanupRuntimeProgram(pb, outputs);
- }
-
- /**
- *
- * @param pb
- * @param outputs
- */
- private static void rCleanupRuntimeProgram( ProgramBlock pb, String[] outputs )
- {
- if( pb instanceof WhileProgramBlock )
- {
- WhileProgramBlock wpb = (WhileProgramBlock)pb;
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- rCleanupRuntimeProgram(pbc,outputs);
- }
- else if( pb instanceof IfProgramBlock )
- {
- IfProgramBlock ipb = (IfProgramBlock)pb;
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- rCleanupRuntimeProgram(pbc,outputs);
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- rCleanupRuntimeProgram(pbc,outputs);
- }
- else if( pb instanceof ForProgramBlock )
- {
- ForProgramBlock fpb = (ForProgramBlock)pb;
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- rCleanupRuntimeProgram(pbc,outputs);
- }
- else
- {
- ArrayList<Instruction> tmp = pb.getInstructions();
- for( int i=0; i<tmp.size(); i++ )
- {
- Instruction linst = tmp.get(i);
- if( linst instanceof VariableCPInstruction && ((VariableCPInstruction)linst).isRemoveVariable() )
- {
- VariableCPInstruction varinst = (VariableCPInstruction) linst;
- for( String var : outputs )
- if( varinst.isRemoveVariable(var) )
- {
- tmp.remove(i);
- i--;
- break;
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/jmlc/PreparedScript.java b/src/main/java/com/ibm/bi/dml/api/jmlc/PreparedScript.java
deleted file mode 100644
index c3d4555..0000000
--- a/src/main/java/com/ibm/bi/dml/api/jmlc/PreparedScript.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api.jmlc;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- * JMLC (Java Machine Learning Connector) API:
- *
- * NOTE: Currently fused API and implementation in order to reduce complexity.
- */
-public class PreparedScript
-{
-
- //input/output specification
- private HashSet<String> _inVarnames = null;
- private HashSet<String> _outVarnames = null;
-
- //internal state (reused)
- private Program _prog = null;
- private LocalVariableMap _vars = null;
-
- /**
- * Meant to be invoked only from Connection
- */
- protected PreparedScript( Program prog, String[] inputs, String[] outputs )
- {
- _prog = prog;
- _vars = new LocalVariableMap();
-
- //populate input/output vars
- _inVarnames = new HashSet<String>();
- for( String var : inputs )
- _inVarnames.add( var );
- _outVarnames = new HashSet<String>();
- for( String var : outputs )
- _outVarnames.add( var );
- }
-
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, ScalarObject scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- _vars.put(varname, scalar);
- }
-
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, boolean scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- BooleanObject bo = new BooleanObject(varname, scalar);
- _vars.put(varname, bo);
- }
-
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, long scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- IntObject io = new IntObject(varname, scalar);
- _vars.put(varname, io);
- }
-
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, double scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- DoubleObject doo = new DoubleObject(varname, scalar);
- _vars.put(varname, doo);
- }
-
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, String scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- StringObject so = new StringObject(varname, scalar);
- _vars.put(varname, so);
- }
-
- /**
- *
- * @param varname
- * @param matrix
- * @throws DMLException
- */
- public void setMatrix(String varname, MatrixBlock matrix)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
-
- DMLConfig conf = ConfigurationManager.getConfig();
- String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
- int blocksize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
-
- //create new matrix object
- MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE, scratch_space+"/"+varname, meta);
- mo.acquireModify(matrix);
- mo.release();
-
- //put create matrix wrapper into symbol table
- _vars.put(varname, mo);
- }
-
- /**
- *
- * @param varname
- * @param matrix
- * @throws DMLException
- */
- public void setMatrix(String varname, double[][] matrix)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- MatrixBlock mb = DataConverter.convertToMatrixBlock(matrix);
- setMatrix(varname, mb);
- }
-
-
- /**
- *
- */
- public void clearParameters()
- {
- _vars.removeAll();
- }
-
- /**
- *
- * @return
- * @throws DMLException
- */
- public ResultVariables executeScript()
- throws DMLException
- {
- //create and populate execution context
- ExecutionContext ec = ExecutionContextFactory.createContext(_prog);
- ec.setVariables(_vars);
-
- //core execute runtime program
- _prog.execute( ec );
-
- //construct results
- ResultVariables rvars = new ResultVariables();
- for( String ovar : _outVarnames )
- if( _vars.keySet().contains(ovar) )
- rvars.addResult(ovar, _vars.get(ovar));
-
- return rvars;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/jmlc/ResultVariables.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/jmlc/ResultVariables.java b/src/main/java/com/ibm/bi/dml/api/jmlc/ResultVariables.java
deleted file mode 100644
index dedb08f..0000000
--- a/src/main/java/com/ibm/bi/dml/api/jmlc/ResultVariables.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api.jmlc;
-
-import java.util.HashMap;
-import java.util.Set;
-
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- * JMLC (Java Machine Learning Connector) API:
- *
- * NOTE: Currently fused API and implementation in order to reduce complexity.
- */
-public class ResultVariables
-{
-
- private HashMap<String, Data> _out = null;
-
- public ResultVariables()
- {
- _out = new HashMap<String, Data>();
- }
-
- public Set<String> getVariableNames()
- {
- return _out.keySet();
- }
-
- public int size()
- {
- return _out.size();
- }
-
- /**
- *
- * @param var
- * @return
- * @throws DMLException
- */
- public double[][] getMatrix(String varname)
- throws DMLException
- {
- if( !_out.containsKey(varname) )
- throw new DMLException("Non-existing output variable: "+varname);
-
- double[][] ret = null;
- Data dat = _out.get(varname);
-
- //basic checks for data type
- if( !(dat instanceof MatrixObject) )
- throw new DMLException("Expected matrix result '"+varname+"' not a matrix.");
-
- //convert output matrix to double array
- MatrixObject mo = (MatrixObject)dat;
- MatrixBlock mb = mo.acquireRead();
- ret = DataConverter.convertToDoubleMatrix(mb);
- mo.release();
-
- return ret;
- }
-
- /**
- *
- *
- * @param ovar
- * @param data
- */
- protected void addResult(String ovar, Data data)
- {
- _out.put(ovar, data);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegression.java b/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegression.java
deleted file mode 100644
index 8803d76..0000000
--- a/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegression.java
+++ /dev/null
@@ -1,464 +0,0 @@
-package com.ibm.bi.dml.api.ml;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.spark.SparkContext;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.ml.classification.LogisticRegressionParams;
-import org.apache.spark.ml.classification.ProbabilisticClassifier;
-import org.apache.spark.ml.param.BooleanParam;
-import org.apache.spark.ml.param.DoubleParam;
-import org.apache.spark.ml.param.IntParam;
-import org.apache.spark.ml.param.StringArrayParam;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.SQLContext;
-
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
-import com.ibm.bi.dml.api.ml.LogisticRegressionModel;
-import com.ibm.bi.dml.api.ml.functions.ConvertSingleColumnToString;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-
-/**
- *
- * This class shows how SystemML can be integrated into MLPipeline. Note, it has not been optimized for performance and
- * is implemented as a proof of concept. An optimized pipeline can be constructed by usage of DML's 'parfor' construct.
- *
- * TODO:
- * - Please note that this class expects 1-based labels. To run below example,
- * please set environment variable 'SYSTEMML_HOME' and create folder 'algorithms'
- * and place atleast two scripts in that folder 'MultiLogReg.dml' and 'GLM-predict.dml'
- * - It is not yet optimized for performance.
- * - Also, it needs to be extended to surface all the parameters of MultiLogReg.dml
- *
- * Example usage:
- * <pre><code>
- * // Code to demonstrate usage of pipeline
- * import org.apache.spark.ml.Pipeline
- * import com.ibm.bi.dml.api.ml.LogisticRegression
- * import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
- * import org.apache.spark.mllib.linalg.Vector
- * case class LabeledDocument(id: Long, text: String, label: Double)
- * case class Document(id: Long, text: String)
- * val training = sc.parallelize(Seq(
- * LabeledDocument(0L, "a b c d e spark", 1.0),
- * LabeledDocument(1L, "b d", 2.0),
- * LabeledDocument(2L, "spark f g h", 1.0),
- * LabeledDocument(3L, "hadoop mapreduce", 2.0),
- * LabeledDocument(4L, "b spark who", 1.0),
- * LabeledDocument(5L, "g d a y", 2.0),
- * LabeledDocument(6L, "spark fly", 1.0),
- * LabeledDocument(7L, "was mapreduce", 2.0),
- * LabeledDocument(8L, "e spark program", 1.0),
- * LabeledDocument(9L, "a e c l", 2.0),
- * LabeledDocument(10L, "spark compile", 1.0),
- * LabeledDocument(11L, "hadoop software", 2.0)))
- * val tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words")
- * val hashingTF = new HashingTF().setNumFeatures(1000).setInputCol(tokenizer.getOutputCol).setOutputCol("features")
- * val lr = new LogisticRegression(sc, sqlContext)
- * val pipeline = new Pipeline().setStages(Array(tokenizer, hashingTF, lr))
- * val model = pipeline.fit(training.toDF)
- * val test = sc.parallelize(Seq(
- * Document(12L, "spark i j k"),
- * Document(13L, "l m n"),
- * Document(14L, "mapreduce spark"),
- * Document(15L, "apache hadoop")))
- * model.transform(test.toDF).show
- *
- * // Code to demonstrate usage of cross-validation
- * import org.apache.spark.ml.Pipeline
- * import com.ibm.bi.dml.api.ml.LogisticRegression
- * import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
- * import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
- * import org.apache.spark.ml.tuning.{ParamGridBuilder, CrossValidator}
- * import org.apache.spark.mllib.linalg.Vector
- * case class LabeledDocument(id: Long, text: String, label: Double)
- * case class Document(id: Long, text: String)
- * val training = sc.parallelize(Seq(
- * LabeledDocument(0L, "a b c d e spark", 1.0),
- * LabeledDocument(1L, "b d", 2.0),
- * LabeledDocument(2L, "spark f g h", 1.0),
- * LabeledDocument(3L, "hadoop mapreduce", 2.0),
- * LabeledDocument(4L, "b spark who", 1.0),
- * LabeledDocument(5L, "g d a y", 2.0),
- * LabeledDocument(6L, "spark fly", 1.0),
- * LabeledDocument(7L, "was mapreduce", 2.0),
- * LabeledDocument(8L, "e spark program", 1.0),
- * LabeledDocument(9L, "a e c l", 2.0),
- * LabeledDocument(10L, "spark compile", 1.0),
- * LabeledDocument(11L, "hadoop software", 2.0)))
- * val tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words")
- * val hashingTF = new HashingTF().setNumFeatures(1000).setInputCol(tokenizer.getOutputCol).setOutputCol("features")
- * val lr = new LogisticRegression(sc, sqlContext)
- * val pipeline = new Pipeline().setStages(Array(tokenizer, hashingTF, lr))
- * val crossval = new CrossValidator().setEstimator(pipeline).setEvaluator(new BinaryClassificationEvaluator)
- * val paramGrid = new ParamGridBuilder().addGrid(hashingTF.numFeatures, Array(10, 100, 1000)).addGrid(lr.regParam, Array(0.1, 0.01)).build()
- * crossval.setEstimatorParamMaps(paramGrid)
- * crossval.setNumFolds(2)
- * val cvModel = crossval.fit(training.toDF)
- * val test = sc.parallelize(Seq(
- * Document(12L, "spark i j k"),
- * Document(13L, "l m n"),
- * Document(14L, "mapreduce spark"),
- * Document(15L, "apache hadoop")))
- * cvModel.transform(test.toDF).show
- * </code></pre>
- *
- */
-public class LogisticRegression extends ProbabilisticClassifier<Vector, LogisticRegression, LogisticRegressionModel>
- implements LogisticRegressionParams {
-
- private static final long serialVersionUID = 7763813395635870734L;
-
- private SparkContext sc = null;
- private SQLContext sqlContext = null;
- private HashMap<String, String> cmdLineParams = new HashMap<String, String>();
-
- private IntParam icpt = new IntParam(this, "icpt", "Value of intercept");
- private DoubleParam reg = new DoubleParam(this, "reg", "Value of regularization parameter");
- private DoubleParam tol = new DoubleParam(this, "tol", "Value of tolerance");
- private IntParam moi = new IntParam(this, "moi", "Max outer iterations");
- private IntParam mii = new IntParam(this, "mii", "Max inner iterations");
- private IntParam labelIndex = new IntParam(this, "li", "Index of the label column");
- private StringArrayParam inputCol = new StringArrayParam(this, "icname", "Feature column name");
- private StringArrayParam outputCol = new StringArrayParam(this, "ocname", "Label column name");
- private int intMin = Integer.MIN_VALUE;
- @SuppressWarnings("unused")
- private int li = 0;
- private String[] icname = new String[1];
- private String[] ocname = new String[1];
-
- public LogisticRegression() {
- }
-
- public LogisticRegression(String uid) {
- }
-
- @Override
- public LogisticRegression copy(org.apache.spark.ml.param.ParamMap paramMap) {
- try {
- // Copy deals with command-line parameter of script MultiLogReg.dml
- LogisticRegression lr = new LogisticRegression(sc, sqlContext);
- lr.cmdLineParams.put(icpt.name(), paramMap.getOrElse(icpt, 0).toString());
- lr.cmdLineParams.put(reg.name(), paramMap.getOrElse(reg, 0.0f).toString());
- lr.cmdLineParams.put(tol.name(), paramMap.getOrElse(tol, 0.000001f).toString());
- lr.cmdLineParams.put(moi.name(), paramMap.getOrElse(moi, 100).toString());
- lr.cmdLineParams.put(mii.name(), paramMap.getOrElse(mii, 0).toString());
-
- return lr;
- } catch (DMLRuntimeException e) {
- e.printStackTrace();
- }
- return null;
-
- }
-
- public LogisticRegression(SparkContext sc, SQLContext sqlContext) throws DMLRuntimeException {
- this.sc = sc;
- this.sqlContext = sqlContext;
-
- setDefault(intercept(), 0);
- cmdLineParams.put(icpt.name(), "0");
- setDefault(regParam(), 0.0f);
- cmdLineParams.put(reg.name(), "0.0f");
- setDefault(tol(), 0.000001f);
- cmdLineParams.put(tol.name(), "0.000001f");
- setDefault(maxOuterIter(), 100);
- cmdLineParams.put(moi.name(), "100");
- setDefault(maxInnerIter(), 0);
- cmdLineParams.put(mii.name(), "0");
- setDefault(labelIdx(), intMin);
- li = intMin;
- setDefault(inputCol(), icname);
- icname[0] = "";
- setDefault(outputCol(), ocname);
- ocname[0] = "";
- }
-
- public LogisticRegression(SparkContext sc, SQLContext sqlContext, int icpt, double reg, double tol, int moi, int mii) throws DMLRuntimeException {
- this.sc = sc;
- this.sqlContext = sqlContext;
-
- setDefault(intercept(), icpt);
- cmdLineParams.put(this.icpt.name(), Integer.toString(icpt));
- setDefault(regParam(), reg);
- cmdLineParams.put(this.reg.name(), Double.toString(reg));
- setDefault(tol(), tol);
- cmdLineParams.put(this.tol.name(), Double.toString(tol));
- setDefault(maxOuterIter(), moi);
- cmdLineParams.put(this.moi.name(), Integer.toString(moi));
- setDefault(maxInnerIter(), mii);
- cmdLineParams.put(this.mii.name(), Integer.toString(mii));
- setDefault(labelIdx(), intMin);
- li = intMin;
- setDefault(inputCol(), icname);
- icname[0] = "";
- setDefault(outputCol(), ocname);
- ocname[0] = "";
- }
-
- @Override
- public String uid() {
- return Long.toString(LogisticRegression.serialVersionUID);
- }
-
- public LogisticRegression setRegParam(double value) {
- cmdLineParams.put(reg.name(), Double.toString(value));
- return (LogisticRegression) setDefault(reg, value);
- }
-
- @Override
- public org.apache.spark.sql.types.StructType validateAndTransformSchema(org.apache.spark.sql.types.StructType arg0, boolean arg1, org.apache.spark.sql.types.DataType arg2) {
- return null;
- }
-
- @Override
- public double getRegParam() {
- return Double.parseDouble(cmdLineParams.get(reg.name()));
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasRegParam$_setter_$regParam_$eq(DoubleParam arg0) {
-
- }
-
- @Override
- public DoubleParam regParam() {
- return reg;
- }
-
- @Override
- public DoubleParam elasticNetParam() {
- return null;
- }
-
- @Override
- public double getElasticNetParam() {
- return 0.0f;
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasElasticNetParam$_setter_$elasticNetParam_$eq(DoubleParam arg0) {
-
- }
-
- @Override
- public int getMaxIter() {
- return 0;
- }
-
- @Override
- public IntParam maxIter() {
- return null;
- }
-
- public LogisticRegression setMaxOuterIter(int value) {
- cmdLineParams.put(moi.name(), Integer.toString(value));
- return (LogisticRegression) setDefault(moi, value);
- }
-
- public int getMaxOuterIter() {
- return Integer.parseInt(cmdLineParams.get(moi.name()));
- }
-
- public IntParam maxOuterIter() {
- return this.moi;
- }
-
- public LogisticRegression setMaxInnerIter(int value) {
- cmdLineParams.put(mii.name(), Integer.toString(value));
- return (LogisticRegression) setDefault(mii, value);
- }
-
- public int getMaxInnerIter() {
- return Integer.parseInt(cmdLineParams.get(mii.name()));
- }
-
- public IntParam maxInnerIter() {
- return mii;
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasMaxIter$_setter_$maxIter_$eq(IntParam arg0) {
-
- }
-
- public LogisticRegression setIntercept(int value) {
- cmdLineParams.put(icpt.name(), Integer.toString(value));
- return (LogisticRegression) setDefault(icpt, value);
- }
-
- public int getIntercept() {
- return Integer.parseInt(cmdLineParams.get(icpt.name()));
- }
-
- public IntParam intercept() {
- return icpt;
- }
-
- @Override
- public BooleanParam fitIntercept() {
- return null;
- }
-
- @Override
- public boolean getFitIntercept() {
- return false;
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasFitIntercept$_setter_$fitIntercept_$eq(BooleanParam arg0) {
-
- }
-
- public LogisticRegression setTol(double value) {
- cmdLineParams.put(tol.name(), Double.toString(value));
- return (LogisticRegression) setDefault(tol, value);
- }
-
- @Override
- public double getTol() {
- return Double.parseDouble(cmdLineParams.get(tol.name()));
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasTol$_setter_$tol_$eq(DoubleParam arg0) {
-
- }
-
- @Override
- public DoubleParam tol() {
- return tol;
- }
-
- @Override
- public double getThreshold() {
- return 0;
- }
-
- @Override
- public void org$apache$spark$ml$param$shared$HasThreshold$_setter_$threshold_$eq(DoubleParam arg0) {
-
- }
-
- @Override
- public DoubleParam threshold() {
- return null;
- }
-
- public LogisticRegression setLabelIndex(int value) {
- li = value;
- return (LogisticRegression) setDefault(labelIndex, value);
- }
-
- public int getLabelIndex() {
- return Integer.parseInt(cmdLineParams.get(labelIndex.name()));
- }
-
- public IntParam labelIdx() {
- return labelIndex;
- }
-
- public LogisticRegression setInputCol(String[] value) {
- icname[0] = value[0];
- return (LogisticRegression) setDefault(inputCol, value);
- }
-
- public String getInputCol() {
- return icname[0];
- }
-
- public StringArrayParam inputCol() {
- return inputCol;
- }
-
- public LogisticRegression setOutputCol(String[] value) {
- ocname[0] = value[0];
- return (LogisticRegression) setDefault(outputCol, value);
- }
-
- public String getOutputCol() {
- return ocname[0];
- }
-
- public StringArrayParam outputCol() {
- return outputCol;
- }
-
- @Override
- public LogisticRegressionModel train(DataFrame df) {
- MLContext ml = null;
- MLOutput out = null;
-
- try {
- ml = new MLContext(this.sc);
- } catch (DMLRuntimeException e1) {
- e1.printStackTrace();
- return null;
- }
-
- // Convert input data to format that SystemML accepts
- MatrixCharacteristics mcXin = new MatrixCharacteristics();
- JavaPairRDD<MatrixIndexes, MatrixBlock> Xin;
- try {
- Xin = RDDConverterUtilsExt.vectorDataFrameToBinaryBlock(new JavaSparkContext(this.sc), df, mcXin, false, "features");
- } catch (DMLRuntimeException e1) {
- e1.printStackTrace();
- return null;
- }
-
- JavaRDD<String> yin = df.select("label").rdd().toJavaRDD().map(new ConvertSingleColumnToString());
-
- try {
- // Register the input/output variables of script 'MultiLogReg.dml'
- ml.registerInput("X", Xin, mcXin);
- ml.registerInput("Y_vec", yin, "csv");
- ml.registerOutput("B_out");
-
- // Or add ifdef in MultiLogReg.dml
- cmdLineParams.put("X", " ");
- cmdLineParams.put("Y", " ");
- cmdLineParams.put("B", " ");
-
-
- // ------------------------------------------------------------------------------------
- // Please note that this logic is subject to change and is put as a placeholder
- String systemmlHome = System.getenv("SYSTEMML_HOME");
- if(systemmlHome == null) {
- System.err.println("ERROR: The environment variable SYSTEMML_HOME is not set.");
- return null;
- }
-
- String dmlFilePath = systemmlHome + File.separator + "algorithms" + File.separator + "MultiLogReg.dml";
- // ------------------------------------------------------------------------------------
-
- synchronized(MLContext.class) {
- // static synchronization is necessary before execute call
- out = ml.execute(dmlFilePath, cmdLineParams);
- }
-
- JavaPairRDD<MatrixIndexes, MatrixBlock> b_out = out.getBinaryBlockedRDD("B_out");
- MatrixCharacteristics b_outMC = out.getMatrixCharacteristics("B_out");
- return new LogisticRegressionModel(b_out, b_outMC, sc).setParent(this);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException(e);
- } catch (DMLException e) {
- throw new RuntimeException(e);
- } catch (ParseException e) {
- throw new RuntimeException(e);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegressionModel.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegressionModel.java b/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegressionModel.java
deleted file mode 100644
index 590372e..0000000
--- a/src/main/java/com/ibm/bi/dml/api/ml/LogisticRegressionModel.java
+++ /dev/null
@@ -1,169 +0,0 @@
-package com.ibm.bi.dml.api.ml;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.spark.SparkContext;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.ml.classification.ProbabilisticClassificationModel;
-import org.apache.spark.ml.param.ParamMap;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
-
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-
-public class LogisticRegressionModel extends ProbabilisticClassificationModel<Vector, LogisticRegressionModel> {
-
- private static final long serialVersionUID = -6464693773946415027L;
- private JavaPairRDD<MatrixIndexes, MatrixBlock> b_out;
- private SparkContext sc;
- private MatrixCharacteristics b_outMC;
- @Override
- public LogisticRegressionModel copy(ParamMap paramMap) {
- return this;
- }
-
- public LogisticRegressionModel(JavaPairRDD<MatrixIndexes, MatrixBlock> b_out2, MatrixCharacteristics b_outMC, SparkContext sc) {
- this.b_out = b_out2;
- this.b_outMC = b_outMC;
- this.sc = sc;
- //this.cmdLineParams = cmdLineParams;
- }
-
- public LogisticRegressionModel() {
- }
-
- public LogisticRegressionModel(String uid) {
- }
-
- @Override
- public String uid() {
- return Long.toString(LogisticRegressionModel.serialVersionUID);
- }
-
- @Override
- public Vector raw2probabilityInPlace(Vector arg0) {
- return arg0;
- }
-
- @Override
- public int numClasses() {
- return 2;
- }
-
- @Override
- public Vector predictRaw(Vector arg0) {
- return arg0;
- }
-
-
- @Override
- public double predict(Vector features) {
- return super.predict(features);
- }
-
- @Override
- public double raw2prediction(Vector rawPrediction) {
- return super.raw2prediction(rawPrediction);
- }
-
- @Override
- public double probability2prediction(Vector probability) {
- return super.probability2prediction(probability);
- }
-
- public static class ConvertIntToRow implements Function<Integer, Row> {
-
- private static final long serialVersionUID = -3480953015655773622L;
-
- @Override
- public Row call(Integer arg0) throws Exception {
- Object[] row_fields = new Object[1];
- row_fields[0] = new Double(arg0);
- return RowFactory.create(row_fields);
- }
-
- }
-
- @Override
- public DataFrame transform(DataFrame dataset) {
- try {
- MatrixCharacteristics mcXin = new MatrixCharacteristics();
- JavaPairRDD<MatrixIndexes, MatrixBlock> Xin;
- try {
- Xin = RDDConverterUtilsExt.vectorDataFrameToBinaryBlock(new JavaSparkContext(this.sc), dataset, mcXin, false, "features");
- } catch (DMLRuntimeException e1) {
- e1.printStackTrace();
- return null;
- }
- MLContext ml = new MLContext(sc);
- ml.registerInput("X", Xin, mcXin);
- ml.registerInput("B_full", b_out, b_outMC); // Changed MLContext for this method
- ml.registerOutput("means");
- HashMap<String, String> param = new HashMap<String, String>();
- param.put("dfam", "3");
-
- // ------------------------------------------------------------------------------------
- // Please note that this logic is subject to change and is put as a placeholder
- String systemmlHome = System.getenv("SYSTEMML_HOME");
- if(systemmlHome == null) {
- System.err.println("ERROR: The environment variable SYSTEMML_HOME is not set.");
- return null;
- }
- // Or add ifdef in GLM-predict.dml
- param.put("X", " ");
- param.put("B", " ");
-
- String dmlFilePath = systemmlHome + File.separator + "algorithms" + File.separator + "GLM-predict.dml";
- // ------------------------------------------------------------------------------------
- MLOutput out = ml.execute(dmlFilePath, param);
-
- SQLContext sqlContext = new SQLContext(sc);
- DataFrame prob = out.getDF(sqlContext, "means", true).withColumnRenamed("C1", "probability");
-
- MLContext mlNew = new MLContext(sc);
- mlNew.registerInput("X", Xin, mcXin);
- mlNew.registerInput("B_full", b_out, b_outMC);
- mlNew.registerInput("Prob", out.getBinaryBlockedRDD("means"), out.getMatrixCharacteristics("means"));
- mlNew.registerOutput("Prediction");
- mlNew.registerOutput("rawPred");
- MLOutput outNew = mlNew.executeScript("Prob = read(\"temp1\"); "
- + "Prediction = rowIndexMax(Prob); "
- + "write(Prediction, \"tempOut\", \"csv\")"
- + "X = read(\"temp2\");"
- + "B_full = read(\"temp3\");"
- + "rawPred = 1 / (1 + exp(- X * t(B_full)) );" // Raw prediction logic:
- + "write(rawPred, \"tempOut1\", \"csv\")");
-
- // TODO: Perform joins in the DML
- DataFrame pred = outNew.getDF(sqlContext, "Prediction").withColumnRenamed("C1", "prediction").withColumnRenamed("ID", "ID1");
- DataFrame rawPred = outNew.getDF(sqlContext, "rawPred", true).withColumnRenamed("C1", "rawPrediction").withColumnRenamed("ID", "ID2");
- DataFrame predictionsNProb = prob.join(pred, prob.col("ID").equalTo(pred.col("ID1"))).select("ID", "probability", "prediction");
- predictionsNProb = predictionsNProb.join(rawPred, predictionsNProb.col("ID").equalTo(rawPred.col("ID2"))).select("ID", "probability", "prediction", "rawPrediction");
- DataFrame dataset1 = RDDConverterUtilsExt.addIDToDataFrame(dataset, sqlContext, "ID");
- return dataset1.join(predictionsNProb, dataset1.col("ID").equalTo(predictionsNProb.col("ID"))).orderBy("id");
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException(e);
- } catch (DMLException e) {
- throw new RuntimeException(e);
- } catch (ParseException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertSingleColumnToString.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertSingleColumnToString.java b/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertSingleColumnToString.java
deleted file mode 100644
index 32844de..0000000
--- a/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertSingleColumnToString.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package com.ibm.bi.dml.api.ml.functions;
-
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.sql.Row;
-
-public class ConvertSingleColumnToString implements Function<Row, String> {
-
- private static final long serialVersionUID = -499763403738768970L;
-
- @Override
- public String call(Row row) throws Exception {
- return row.apply(0).toString();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertVectorToDouble.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertVectorToDouble.java b/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertVectorToDouble.java
deleted file mode 100644
index e9c77ba..0000000
--- a/src/main/java/com/ibm/bi/dml/api/ml/functions/ConvertVectorToDouble.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package com.ibm.bi.dml.api.ml.functions;
-
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.sql.Row;
-
-public class ConvertVectorToDouble implements Function<Row, Double> {
-
- private static final long serialVersionUID = -6612447783777073929L;
-
- @Override
- public Double call(Row row) throws Exception {
-
- return row.getDouble(0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/monitoring/InstructionComparator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/monitoring/InstructionComparator.java b/src/main/java/com/ibm/bi/dml/api/monitoring/InstructionComparator.java
deleted file mode 100644
index 906ebb4..0000000
--- a/src/main/java/com/ibm/bi/dml/api/monitoring/InstructionComparator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package com.ibm.bi.dml.api.monitoring;
-
-import java.util.Comparator;
-import java.util.HashMap;
-
-public class InstructionComparator implements Comparator<String>{
-
- HashMap<String, Long> instructionCreationTime;
- public InstructionComparator(HashMap<String, Long> instructionCreationTime) {
- this.instructionCreationTime = instructionCreationTime;
- }
- @Override
- public int compare(String o1, String o2) {
- try {
- return instructionCreationTime.get(o1).compareTo(instructionCreationTime.get(o2));
- }
- catch(Exception e) {
- return -1;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/monitoring/Location.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/monitoring/Location.java b/src/main/java/com/ibm/bi/dml/api/monitoring/Location.java
deleted file mode 100644
index 3cc2957..0000000
--- a/src/main/java/com/ibm/bi/dml/api/monitoring/Location.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package com.ibm.bi.dml.api.monitoring;
-
-public class Location implements Comparable<Location> {
- public int beginLine;
- public int endLine;
- public int beginCol;
- public int endCol;
- public Location(int beginLine, int endLine, int beginCol, int endCol) {
- this.beginLine = beginLine;
- this.endLine = endLine;
- this.beginCol = beginCol;
- this.endCol = endCol;
- }
-
- @Override
- public boolean equals(Object other) {
- if(other instanceof Location) {
- Location loc = (Location) other;
- if(loc.beginLine == beginLine && loc.endLine == endLine && loc.beginCol == beginCol && loc.endCol == endCol) {
- return true;
- }
- else
- return false;
- }
- return false;
- }
-
- private int compare(int v1, int v2) {
- return new Integer(v1).compareTo(new Integer(v2));
- }
-
- public String toString() {
- return beginLine + ":" + beginCol + ", " + endLine + ":" + endCol;
- }
-
- @Override
- public int compareTo(Location loc) {
- if(loc.beginLine == beginLine && loc.endLine == endLine && loc.beginCol == beginCol && loc.endCol == endCol)
- return 0;
-
- int retVal = compare(beginLine, loc.beginLine);
- if(retVal != 0) {
- return retVal;
- }
- else {
- retVal = compare(beginCol, loc.beginCol);
- if(retVal != 0) {
- return retVal;
- }
- else {
- retVal = compare(endLine, loc.endLine);
- if(retVal != 0) {
- return retVal;
- }
- else {
- return compare(endCol, loc.endCol);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/monitoring/SparkMonitoringUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/monitoring/SparkMonitoringUtil.java b/src/main/java/com/ibm/bi/dml/api/monitoring/SparkMonitoringUtil.java
deleted file mode 100644
index 9b4996b..0000000
--- a/src/main/java/com/ibm/bi/dml/api/monitoring/SparkMonitoringUtil.java
+++ /dev/null
@@ -1,602 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package com.ibm.bi.dml.api.monitoring;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map.Entry;
-
-import scala.collection.Seq;
-import scala.xml.Node;
-
-import com.google.common.collect.Multimap;
-import com.google.common.collect.TreeMultimap;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.SparkListener;
-
-/**
- * Usage guide:
- * MLContext mlCtx = new MLContext(sc, true);
- * mlCtx.register...
- * mlCtx.execute(...)
- * mlCtx.getMonitoringUtil().getRuntimeInfoInHTML("runtime.html");
- */
-public class SparkMonitoringUtil {
- // ----------------------------------------------------
- // For VLDB Demo:
- private Multimap<Location, String> instructions = TreeMultimap.create();
- private Multimap<String, Integer> stageIDs = TreeMultimap.create(); // instruction -> stageIds
- private Multimap<String, Integer> jobIDs = TreeMultimap.create(); // instruction -> jobIds
- private HashMap<String, String> lineageInfo = new HashMap<String, String>(); // instruction -> lineageInfo
- private HashMap<String, Long> instructionCreationTime = new HashMap<String, Long>();
-
- private Multimap<Integer, String> rddInstructionMapping = TreeMultimap.create();
-
- private HashSet<String> getRelatedInstructions(int stageID) {
- HashSet<String> retVal = new HashSet<String>();
- if(_sparkListener != null) {
- ArrayList<Integer> rdds = _sparkListener.stageRDDMapping.get(stageID);
- for(Integer rddID : rdds) {
- retVal.addAll(rddInstructionMapping.get(rddID));
- }
- }
- return retVal;
- }
-
- private SparkListener _sparkListener = null;
- public SparkListener getSparkListener() {
- return _sparkListener;
- }
-
- private String explainOutput = "";
-
- public String getExplainOutput() {
- return explainOutput;
- }
-
- public void setExplainOutput(String explainOutput) {
- this.explainOutput = explainOutput;
- }
-
- public SparkMonitoringUtil(SparkListener sparkListener) {
- _sparkListener = sparkListener;
- }
-
- public void addCurrentInstruction(SPInstruction inst) {
- if(_sparkListener != null) {
- _sparkListener.addCurrentInstruction(inst);
- }
- }
-
- public void addRDDForInstruction(SPInstruction inst, Integer rddID) {
- this.rddInstructionMapping.put(rddID, getInstructionString(inst));
- }
-
- public void removeCurrentInstruction(SPInstruction inst) {
- if(_sparkListener != null) {
- _sparkListener.removeCurrentInstruction(inst);
- }
- }
-
- public void setDMLString(String dmlStr) {
- this.dmlStrForMonitoring = dmlStr;
- }
-
- public void resetMonitoringData() {
- if(_sparkListener != null && _sparkListener.stageDAGs != null)
- _sparkListener.stageDAGs.clear();
- if(_sparkListener != null && _sparkListener.stageTimeline != null)
- _sparkListener.stageTimeline.clear();
- }
-
- // public Multimap<Location, String> hops = ArrayListMultimap.create(); TODO:
- private String dmlStrForMonitoring = null;
- public void getRuntimeInfoInHTML(String htmlFilePath) throws DMLRuntimeException, IOException {
- String jsAndCSSFiles = "<script src=\"js/lodash.min.js\"></script>"
- + "<script src=\"js/jquery-1.11.1.min.js\"></script>"
- + "<script src=\"js/d3.min.js\"></script>"
- + "<script src=\"js/bootstrap-tooltip.js\"></script>"
- + "<script src=\"js/dagre-d3.min.js\"></script>"
- + "<script src=\"js/graphlib-dot.min.js\"></script>"
- + "<script src=\"js/spark-dag-viz.js\"></script>"
- + "<script src=\"js/timeline-view.js\"></script>"
- + "<script src=\"js/vis.min.js\"></script>"
- + "<link rel=\"stylesheet\" href=\"css/bootstrap.min.css\">"
- + "<link rel=\"stylesheet\" href=\"css/vis.min.css\">"
- + "<link rel=\"stylesheet\" href=\"css/spark-dag-viz.css\">"
- + "<link rel=\"stylesheet\" href=\"css/timeline-view.css\"> ";
- BufferedWriter bw = new BufferedWriter(new FileWriter(htmlFilePath));
- bw.write("<html><head>\n");
- bw.write(jsAndCSSFiles + "\n");
- bw.write("</head><body>\n<table border=1>\n");
-
- bw.write("<tr>\n");
- bw.write("<td><b>Position in script</b></td>\n");
- bw.write("<td><b>DML</b></td>\n");
- bw.write("<td><b>Instruction</b></td>\n");
- bw.write("<td><b>StageIDs</b></td>\n");
- bw.write("<td><b>RDD Lineage</b></td>\n");
- bw.write("</tr>\n");
-
- for(Location loc : instructions.keySet()) {
- String dml = getExpression(loc);
-
- // Sort the instruction with time - so as to separate recompiled instructions
- List<String> listInst = new ArrayList<String>(instructions.get(loc));
- Collections.sort(listInst, new InstructionComparator(instructionCreationTime));
-
- if(dml != null && dml.trim().length() > 1) {
- bw.write("<tr>\n");
- int rowSpan = listInst.size();
- bw.write("<td rowspan=\"" + rowSpan + "\">" + loc.toString() + "</td>\n");
- bw.write("<td rowspan=\"" + rowSpan + "\">" + dml + "</td>\n");
- boolean firstTime = true;
- for(String inst : listInst) {
- if(!firstTime)
- bw.write("<tr>\n");
-
- if(inst.startsWith("SPARK"))
- bw.write("<td style=\"color:red\">" + inst + "</td>\n");
- else if(isInterestingCP(inst))
- bw.write("<td style=\"color:blue\">" + inst + "</td>\n");
- else
- bw.write("<td>" + inst + "</td>\n");
-
- bw.write("<td>" + getStageIDAsString(inst) + "</td>\n");
- if(lineageInfo.containsKey(inst))
- bw.write("<td>" + lineageInfo.get(inst).replaceAll("\n", "<br />") + "</td>\n");
- else
- bw.write("<td></td>\n");
-
- bw.write("</tr>\n");
- firstTime = false;
- }
-
- }
-
- }
-
- bw.write("</table></body>\n</html>");
- bw.close();
- }
-
- private String getInQuotes(String str) {
- return "\"" + str + "\"";
- }
- private String getEscapedJSON(String json) {
- if(json == null)
- return "";
- else {
- return json
- //.replaceAll("\\\\", "\\\\\\")
- .replaceAll("\\t", "\\\\t")
- .replaceAll("/", "\\\\/")
- .replaceAll("\"", "\\\\\"")
- .replaceAll("\\r?\\n", "\\\\n");
- }
- }
-
- private long maxExpressionExecutionTime = 0;
- HashMap<Integer, Long> stageExecutionTimes = new HashMap<Integer, Long>();
- HashMap<String, Long> expressionExecutionTimes = new HashMap<String, Long>();
- HashMap<String, Long> instructionExecutionTimes = new HashMap<String, Long>();
- HashMap<Integer, HashSet<String>> relatedInstructionsPerStage = new HashMap<Integer, HashSet<String>>();
- private void fillExecutionTimes() {
- stageExecutionTimes.clear();
- expressionExecutionTimes.clear();
- for(Location loc : instructions.keySet()) {
- List<String> listInst = new ArrayList<String>(instructions.get(loc));
- long expressionExecutionTime = 0;
-
- if(listInst != null && listInst.size() > 0) {
- for(String inst : listInst) {
- long instructionExecutionTime = 0;
- for(Integer stageId : stageIDs.get(inst)) {
- try {
- if(getStageExecutionTime(stageId) != null) {
- long stageExecTime = getStageExecutionTime(stageId);
- instructionExecutionTime += stageExecTime;
- expressionExecutionTime += stageExecTime;
- stageExecutionTimes.put(stageId, stageExecTime);
- }
- }
- catch(Exception e) {}
-
- relatedInstructionsPerStage.put(stageId, getRelatedInstructions(stageId));
- }
- instructionExecutionTimes.put(inst, instructionExecutionTime);
- }
- expressionExecutionTime /= listInst.size(); // average
- }
- maxExpressionExecutionTime = Math.max(maxExpressionExecutionTime, expressionExecutionTime);
- expressionExecutionTimes.put(loc.toString(), expressionExecutionTime);
- }
-
- // Now fill empty instructions
- for(Entry<String, Long> kv : instructionExecutionTimes.entrySet()) {
- if(kv.getValue() == 0) {
- // Find all stages that contain this as related instruction
- long sumExecutionTime = 0;
- for(Entry<Integer, HashSet<String>> kv1 : relatedInstructionsPerStage.entrySet()) {
- if(kv1.getValue().contains(kv.getKey())) {
- sumExecutionTime += stageExecutionTimes.get(kv1.getKey());
- }
- }
- kv.setValue(sumExecutionTime);
- }
- }
-
- for(Location loc : instructions.keySet()) {
- if(expressionExecutionTimes.get(loc.toString()) == 0) {
- List<String> listInst = new ArrayList<String>(instructions.get(loc));
- long expressionExecutionTime = 0;
- if(listInst != null && listInst.size() > 0) {
- for(String inst : listInst) {
- expressionExecutionTime += instructionExecutionTimes.get(inst);
- }
- }
- expressionExecutionTime /= listInst.size(); // average
- maxExpressionExecutionTime = Math.max(maxExpressionExecutionTime, expressionExecutionTime);
- expressionExecutionTimes.put(loc.toString(), expressionExecutionTime);
- }
- }
-
- }
-
- /**
- * Useful to avoid passing large String through Py4J
- * @param fileName
- * @throws DMLRuntimeException
- * @throws IOException
- */
- public void saveRuntimeInfoInJSONFormat(String fileName) throws DMLRuntimeException, IOException {
- String json = getRuntimeInfoInJSONFormat();
- BufferedWriter bw = new BufferedWriter(new FileWriter(fileName));
- bw.write(json);
- bw.close();
- }
-
- public String getRuntimeInfoInJSONFormat() throws DMLRuntimeException, IOException {
- StringBuilder retVal = new StringBuilder("{\n");
-
- retVal.append(getInQuotes("dml") + ":" + getInQuotes(getEscapedJSON(dmlStrForMonitoring)) + ",\n");
- retVal.append(getInQuotes("expressions") + ":" + "[\n");
-
- boolean isFirstExpression = true;
- fillExecutionTimes();
-
- for(Location loc : instructions.keySet()) {
- String dml = getEscapedJSON(getExpressionInJSON(loc));
-
- if(dml != null) {
- // Sort the instruction with time - so as to separate recompiled instructions
- List<String> listInst = new ArrayList<String>(instructions.get(loc));
- Collections.sort(listInst, new InstructionComparator(instructionCreationTime));
-
- if(!isFirstExpression) {
- retVal.append(",\n");
- }
- retVal.append("{\n");
- isFirstExpression = false;
-
- retVal.append(getInQuotes("beginLine") + ":" + loc.beginLine + ",\n");
- retVal.append(getInQuotes("beginCol") + ":" + loc.beginCol + ",\n");
- retVal.append(getInQuotes("endLine") + ":" + loc.endLine + ",\n");
- retVal.append(getInQuotes("endCol") + ":" + loc.endCol + ",\n");
-
- long expressionExecutionTime = expressionExecutionTimes.get(loc.toString());
- retVal.append(getInQuotes("expressionExecutionTime") + ":" + expressionExecutionTime + ",\n");
- retVal.append(getInQuotes("expressionHeavyHitterFactor") + ":" + ((double)expressionExecutionTime / (double)maxExpressionExecutionTime) + ",\n");
-
- retVal.append(getInQuotes("expression") + ":" + getInQuotes(dml) + ",\n");
-
- retVal.append(getInQuotes("instructions") + ":" + "[\n");
-
- boolean firstTime = true;
- for(String inst : listInst) {
-
- if(!firstTime)
- retVal.append(", {");
- else
- retVal.append("{");
-
- if(inst.startsWith("SPARK")) {
- retVal.append(getInQuotes("isSpark") + ":" + "true,\n");
- }
- else if(isInterestingCP(inst)) {
- retVal.append(getInQuotes("isInteresting") + ":" + "true,\n");
- }
-
- retVal.append(getStageIDAsJSONString(inst) + "\n");
- if(lineageInfo.containsKey(inst)) {
- retVal.append(getInQuotes("lineageInfo") + ":" + getInQuotes(getEscapedJSON(lineageInfo.get(inst))) + ",\n");
- }
-
- retVal.append(getInQuotes("instruction") + ":" + getInQuotes(getEscapedJSON(inst)));
- retVal.append("}");
- firstTime = false;
- }
-
- retVal.append("]\n");
- retVal.append("}\n");
- }
-
- }
-
- return retVal.append("]\n}").toString();
- }
-
-
- private boolean isInterestingCP(String inst) {
- if(inst.startsWith("CP rmvar") || inst.startsWith("CP cpvar") || inst.startsWith("CP mvvar"))
- return false;
- else if(inst.startsWith("CP"))
- return true;
- else
- return false;
- }
-
- private String getStageIDAsString(String instruction) {
- String retVal = "";
- for(Integer stageId : stageIDs.get(instruction)) {
- String stageDAG = "";
- String stageTimeLine = "";
-
- if(getStageDAGs(stageId) != null) {
- stageDAG = getStageDAGs(stageId).toString();
- }
-
- if(getStageTimeLine(stageId) != null) {
- stageTimeLine = getStageTimeLine(stageId).toString();
- }
-
- retVal += "Stage:" + stageId +
- " ("
- + "<div>"
- + stageDAG.replaceAll("toggleDagViz\\(false\\)", "toggleDagViz(false, this)")
- + "</div>, "
- + "<div id=\"timeline-" + stageId + "\">"
- + stageTimeLine
- .replaceAll("drawTaskAssignmentTimeline\\(", "registerTimelineData(" + stageId + ", ")
- .replaceAll("class=\"expand-task-assignment-timeline\"", "class=\"expand-task-assignment-timeline\" onclick=\"toggleStageTimeline(this)\"")
- + "</div>"
- + ")";
- }
- return retVal;
- }
-
- private String getStageIDAsJSONString(String instruction) {
- long instructionExecutionTime = instructionExecutionTimes.get(instruction);
-
- StringBuilder retVal = new StringBuilder(getInQuotes("instructionExecutionTime") + ":" + instructionExecutionTime + ",\n");
-
- boolean isFirst = true;
- if(stageIDs.get(instruction).size() == 0) {
- // Find back references
- HashSet<Integer> relatedStages = new HashSet<Integer>();
- for(Entry<Integer, HashSet<String>> kv : relatedInstructionsPerStage.entrySet()) {
- if(kv.getValue().contains(instruction)) {
- relatedStages.add(kv.getKey());
- }
- }
- HashSet<String> relatedInstructions = new HashSet<String>();
- for(Entry<String, Integer> kv : stageIDs.entries()) {
- if(relatedStages.contains(kv.getValue())) {
- relatedInstructions.add(kv.getKey());
- }
- }
-
- retVal.append(getInQuotes("backReferences") + ": [\n");
- boolean isFirstRelInst = true;
- for(String relInst : relatedInstructions) {
- if(!isFirstRelInst) {
- retVal.append(",\n");
- }
- retVal.append(getInQuotes(relInst));
- isFirstRelInst = false;
- }
- retVal.append("], \n");
- }
- else {
- retVal.append(getInQuotes("stages") + ": {");
- for(Integer stageId : stageIDs.get(instruction)) {
- String stageDAG = "";
- String stageTimeLine = "";
-
- if(getStageDAGs(stageId) != null) {
- stageDAG = getStageDAGs(stageId).toString();
- }
-
- if(getStageTimeLine(stageId) != null) {
- stageTimeLine = getStageTimeLine(stageId).toString();
- }
-
- long stageExecutionTime = stageExecutionTimes.get(stageId);
- if(!isFirst) {
- retVal.append(",\n");
- }
-
- retVal.append(getInQuotes("" + stageId) + ": {");
-
- // Now add related instructions
- HashSet<String> relatedInstructions = relatedInstructionsPerStage.get(stageId);
-
- retVal.append(getInQuotes("relatedInstructions") + ": [\n");
- boolean isFirstRelInst = true;
- for(String relInst : relatedInstructions) {
- if(!isFirstRelInst) {
- retVal.append(",\n");
- }
- retVal.append(getInQuotes(relInst));
- isFirstRelInst = false;
- }
- retVal.append("],\n");
-
- retVal.append(getInQuotes("DAG") + ":")
- .append(
- getInQuotes(
- getEscapedJSON(stageDAG.replaceAll("toggleDagViz\\(false\\)", "toggleDagViz(false, this)"))
- ) + ",\n"
- )
- .append(getInQuotes("stageExecutionTime") + ":" + stageExecutionTime + ",\n")
- .append(getInQuotes("timeline") + ":")
- .append(
- getInQuotes(
- getEscapedJSON(
- stageTimeLine
- .replaceAll("drawTaskAssignmentTimeline\\(", "registerTimelineData(" + stageId + ", ")
- .replaceAll("class=\"expand-task-assignment-timeline\"", "class=\"expand-task-assignment-timeline\" onclick=\"toggleStageTimeline(this)\""))
- )
- )
- .append("}");
-
- isFirst = false;
- }
- retVal.append("}, ");
- }
-
-
- retVal.append(getInQuotes("jobs") + ": {");
- isFirst = true;
- for(Integer jobId : jobIDs.get(instruction)) {
- String jobDAG = "";
-
- if(getJobDAGs(jobId) != null) {
- jobDAG = getJobDAGs(jobId).toString();
- }
- if(!isFirst) {
- retVal.append(",\n");
- }
-
- retVal.append(getInQuotes("" + jobId) + ": {")
- .append(getInQuotes("DAG") + ":" )
- .append(getInQuotes(
- getEscapedJSON(jobDAG.replaceAll("toggleDagViz\\(true\\)", "toggleDagViz(true, this)"))
- ) + "}\n");
-
- isFirst = false;
- }
- retVal.append("}, ");
-
- return retVal.toString();
- }
-
-
- String [] dmlLines = null;
- private String getExpression(Location loc) {
- try {
- if(dmlLines == null) {
- dmlLines = dmlStrForMonitoring.split("\\r?\\n");
- }
- if(loc.beginLine == loc.endLine) {
- return dmlLines[loc.beginLine-1].substring(loc.beginCol-1, loc.endCol);
- }
- else {
- String retVal = dmlLines[loc.beginLine-1].substring(loc.beginCol-1);
- for(int i = loc.beginLine+1; i < loc.endLine; i++) {
- retVal += "<br />" + dmlLines[i-1];
- }
- retVal += "<br />" + dmlLines[loc.endLine-1].substring(0, loc.endCol);
- return retVal;
- }
- }
- catch(Exception e) {
- return null; // "[[" + loc.beginLine + "," + loc.endLine + "," + loc.beginCol + "," + loc.endCol + "]]";
- }
- }
-
-
- private String getExpressionInJSON(Location loc) {
- try {
- if(dmlLines == null) {
- dmlLines = dmlStrForMonitoring.split("\\r?\\n");
- }
- if(loc.beginLine == loc.endLine) {
- return dmlLines[loc.beginLine-1].substring(loc.beginCol-1, loc.endCol);
- }
- else {
- String retVal = dmlLines[loc.beginLine-1].substring(loc.beginCol-1);
- for(int i = loc.beginLine+1; i < loc.endLine; i++) {
- retVal += "\\n" + dmlLines[i-1];
- }
- retVal += "\\n" + dmlLines[loc.endLine-1].substring(0, loc.endCol);
- return retVal;
- }
- }
- catch(Exception e) {
- return null; // "[[" + loc.beginLine + "," + loc.endLine + "," + loc.beginCol + "," + loc.endCol + "]]";
- }
- }
-
- public Seq<Node> getStageDAGs(int stageIDs) {
- if(_sparkListener == null || _sparkListener.stageDAGs == null)
- return null;
- else
- return _sparkListener.stageDAGs.get(stageIDs);
- }
-
- public Long getStageExecutionTime(int stageID) {
- if(_sparkListener == null || _sparkListener.stageDAGs == null)
- return null;
- else
- return _sparkListener.stageExecutionTime.get(stageID);
- }
-
- public Seq<Node> getJobDAGs(int jobID) {
- if(_sparkListener == null || _sparkListener.jobDAGs == null)
- return null;
- else
- return _sparkListener.jobDAGs.get(jobID);
- }
-
- public Seq<Node> getStageTimeLine(int stageIDs) {
- if(_sparkListener == null || _sparkListener.stageTimeline == null)
- return null;
- else
- return _sparkListener.stageTimeline.get(stageIDs);
- }
- public void setLineageInfo(Instruction inst, String plan) {
- lineageInfo.put(getInstructionString(inst), plan);
- }
- public void setStageId(Instruction inst, int stageId) {
- stageIDs.put(getInstructionString(inst), stageId);
- }
- public void setJobId(Instruction inst, int jobId) {
- jobIDs.put(getInstructionString(inst), jobId);
- }
- public void setInstructionLocation(Location loc, Instruction inst) {
- String instStr = getInstructionString(inst);
- instructions.put(loc, instStr);
- instructionCreationTime.put(instStr, System.currentTimeMillis());
- }
- private String getInstructionString(Instruction inst) {
- String tmp = inst.toString();
- tmp = tmp.replaceAll(Lop.OPERAND_DELIMITOR, " ");
- tmp = tmp.replaceAll(Lop.DATATYPE_PREFIX, ".");
- tmp = tmp.replaceAll(Lop.INSTRUCTION_DELIMITOR, ", ");
- return tmp;
- }
-}
[07/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/SparkExecutionContext.java
deleted file mode 100644
index 48b4699..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/SparkExecutionContext.java
+++ /dev/null
@@ -1,1155 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.context;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.broadcast.Broadcast;
-import org.apache.spark.storage.RDDInfo;
-import org.apache.spark.storage.StorageLevel;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.spark.CheckpointSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BlockPartitioner;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BroadcastObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LineageObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockPairFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyTextInputFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-
-
-public class SparkExecutionContext extends ExecutionContext
-{
-
- private static final Log LOG = LogFactory.getLog(SparkExecutionContext.class.getName());
-
- //internal configurations
- private static boolean LAZY_SPARKCTX_CREATION = true;
- private static boolean ASYNCHRONOUS_VAR_DESTROY = true;
- private static boolean FAIR_SCHEDULER_MODE = true;
-
- //executor memory and relative fractions as obtained from the spark configuration
- private static long _memExecutors = -1; //mem per executors
- private static double _memRatioData = -1;
- private static double _memRatioShuffle = -1;
- private static int _numExecutors = -1; //total executors
- private static int _defaultPar = -1; //total vcores
- private static boolean _confOnly = false; //infrastructure info based on config
-
- // Only one SparkContext may be active per JVM. You must stop() the active SparkContext before creating a new one.
- // This limitation may eventually be removed; see SPARK-2243 for more details.
- private static JavaSparkContext _spctx = null;
-
- protected SparkExecutionContext(Program prog)
- {
- //protected constructor to force use of ExecutionContextFactory
- this( true, prog );
- }
-
- protected SparkExecutionContext(boolean allocateVars, Program prog)
- {
- //protected constructor to force use of ExecutionContextFactory
- super( allocateVars, prog );
-
- //spark context creation via internal initializer
- if( !(LAZY_SPARKCTX_CREATION && OptimizerUtils.isHybridExecutionMode()) ) {
- initSparkContext();
- }
- }
-
- /**
- * Returns the used singleton spark context. In case of lazy spark context
- * creation, this methods blocks until the spark context is created.
- *
- * @return
- */
- public JavaSparkContext getSparkContext()
- {
- //lazy spark context creation on demand (lazy instead of asynchronous
- //to avoid wait for uninitialized spark context on close)
- if( LAZY_SPARKCTX_CREATION ) {
- initSparkContext();
- }
-
- //return the created spark context
- return _spctx;
- }
-
- /**
- *
- * @return
- */
- public static JavaSparkContext getSparkContextStatic()
- {
- initSparkContext();
- return _spctx;
- }
-
- /**
- *
- */
- public void close()
- {
- synchronized( SparkExecutionContext.class ) {
- if( _spctx != null )
- {
- //stop the spark context if existing
- _spctx.stop();
-
- //make sure stopped context is never used again
- _spctx = null;
- }
-
- }
- }
-
- public static boolean isLazySparkContextCreation(){
- return LAZY_SPARKCTX_CREATION;
- }
-
- /**
- *
- */
- private synchronized static void initSparkContext()
- {
- //check for redundant spark context init
- if( _spctx != null )
- return;
-
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- //create a default spark context (master, appname, etc refer to system properties
- //as given in the spark configuration or during spark-submit)
-
- MLContext mlCtx = MLContextProxy.getActiveMLContext();
- if(mlCtx != null)
- {
- // This is when DML is called through spark shell
- // Will clean the passing of static variables later as this involves minimal change to DMLScript
- _spctx = new JavaSparkContext(mlCtx.getSparkContext());
- }
- else
- {
- if(DMLScript.USE_LOCAL_SPARK_CONFIG) {
- // For now set 4 cores for integration testing :)
- SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("My local integration test app");
- // This is discouraged in spark but have added only for those testcase that cannot stop the context properly
- // conf.set("spark.driver.allowMultipleContexts", "true");
- conf.set("spark.ui.enabled", "false");
- _spctx = new JavaSparkContext(conf);
- }
- else //default cluster setup
- {
- //setup systemml-preferred spark configuration (w/o user choice)
- SparkConf conf = new SparkConf();
-
- //always set unlimited result size (required for cp collect)
- conf.set("spark.driver.maxResultSize", "0");
-
- //always use the fair scheduler (for single jobs, it's equivalent to fifo
- //but for concurrent jobs in parfor it ensures better data locality because
- //round robin assignment mitigates the problem of 'sticky slots')
- if( FAIR_SCHEDULER_MODE ) {
- conf.set("spark.scheduler.mode", "FAIR");
- }
-
- _spctx = new JavaSparkContext(conf);
- }
- }
-
- //globally add binaryblock serialization framework for all hdfs read/write operations
- //TODO if spark context passed in from outside (mlcontext), we need to clean this up at the end
- if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
- MRJobConfiguration.addBinaryBlockSerializationFramework( _spctx.hadoopConfiguration() );
-
- //statistics maintenance
- if( DMLScript.STATISTICS ){
- Statistics.setSparkCtxCreateTime(System.nanoTime()-t0);
- }
- }
-
- /**
- * Spark instructions should call this for all matrix inputs except broadcast
- * variables.
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- public JavaPairRDD<MatrixIndexes,MatrixBlock> getBinaryBlockRDDHandleForVariable( String varname )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- return (JavaPairRDD<MatrixIndexes,MatrixBlock>) getRDDHandleForVariable( varname, InputInfo.BinaryBlockInputInfo);
- }
-
- /**
- *
- * @param varname
- * @param inputInfo
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public JavaPairRDD<?,?> getRDDHandleForVariable( String varname, InputInfo inputInfo )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- MatrixObject mo = getMatrixObject(varname);
- return getRDDHandleForMatrixObject(mo, inputInfo);
- }
-
- /**
- * This call returns an RDD handle for a given matrix object. This includes
- * the creation of RDDs for in-memory or binary-block HDFS data.
- *
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- public JavaPairRDD<?,?> getRDDHandleForMatrixObject( MatrixObject mo, InputInfo inputInfo )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //NOTE: MB this logic should be integrated into MatrixObject
- //However, for now we cannot assume that spark libraries are
- //always available and hence only store generic references in
- //matrix object while all the logic is in the SparkExecContext
-
- JavaPairRDD<?,?> rdd = null;
- //CASE 1: rdd already existing (reuse if checkpoint or trigger
- //pending rdd operations if not yet cached but prevent to re-evaluate
- //rdd operations if already executed and cached
- if( mo.getRDDHandle()!=null
- && (mo.getRDDHandle().isCheckpointRDD() || !mo.isCached(false)) )
- {
- //return existing rdd handling (w/o input format change)
- rdd = mo.getRDDHandle().getRDD();
- }
- //CASE 2: dirty in memory data or cached result of rdd operations
- else if( mo.isDirty() || mo.isCached(false) )
- {
- //get in-memory matrix block and parallelize it
- //w/ guarded parallelize (fallback to export, rdd from file if too large)
- boolean fromFile = false;
- if( !OptimizerUtils.checkSparkCollectMemoryBudget(mo.getMatrixCharacteristics(), 0) ) {
- if( mo.isDirty() ) { //write only if necessary
- mo.exportData();
- }
- rdd = getSparkContext().hadoopFile( mo.getFileName(), inputInfo.inputFormatClass, inputInfo.inputKeyClass, inputInfo.inputValueClass);
- rdd = ((JavaPairRDD<MatrixIndexes, MatrixBlock>)rdd).mapToPair( new CopyBlockPairFunction() ); //cp is workaround for read bug
- fromFile = true;
- }
- else { //default case
- MatrixBlock mb = mo.acquireRead(); //pin matrix in memory
- rdd = toJavaPairRDD(getSparkContext(), mb, (int)mo.getNumRowsPerBlock(), (int)mo.getNumColumnsPerBlock());
- mo.release(); //unpin matrix
- }
-
- //keep rdd handle for future operations on it
- RDDObject rddhandle = new RDDObject(rdd, mo.getVarName());
- rddhandle.setHDFSFile(fromFile);
- mo.setRDDHandle(rddhandle);
- }
- //CASE 3: non-dirty (file exists on HDFS)
- else
- {
- // parallelize hdfs-resident file
- // For binary block, these are: SequenceFileInputFormat.class, MatrixIndexes.class, MatrixBlock.class
- if(inputInfo == InputInfo.BinaryBlockInputInfo) {
- rdd = getSparkContext().hadoopFile( mo.getFileName(), inputInfo.inputFormatClass, inputInfo.inputKeyClass, inputInfo.inputValueClass);
- //note: this copy is still required in Spark 1.4 because spark hands out whatever the inputformat
- //recordreader returns; the javadoc explicitly recommend to copy all key/value pairs
- rdd = ((JavaPairRDD<MatrixIndexes, MatrixBlock>)rdd).mapToPair( new CopyBlockPairFunction() ); //cp is workaround for read bug
- }
- else if(inputInfo == InputInfo.TextCellInputInfo || inputInfo == InputInfo.CSVInputInfo || inputInfo == InputInfo.MatrixMarketInputInfo) {
- rdd = getSparkContext().hadoopFile( mo.getFileName(), inputInfo.inputFormatClass, inputInfo.inputKeyClass, inputInfo.inputValueClass);
- rdd = ((JavaPairRDD<LongWritable, Text>)rdd).mapToPair( new CopyTextInputFunction() ); //cp is workaround for read bug
- }
- else if(inputInfo == InputInfo.BinaryCellInputInfo) {
- rdd = getSparkContext().hadoopFile( mo.getFileName(), inputInfo.inputFormatClass, inputInfo.inputKeyClass, inputInfo.inputValueClass);
- rdd = ((JavaPairRDD<MatrixIndexes, MatrixCell>)rdd).mapToPair( new CopyBinaryCellFunction() ); //cp is workaround for read bug
- }
- else {
- throw new DMLRuntimeException("Incorrect input format in getRDDHandleForVariable");
- }
-
- //keep rdd handle for future operations on it
- RDDObject rddhandle = new RDDObject(rdd, mo.getVarName());
- rddhandle.setHDFSFile(true);
- mo.setRDDHandle(rddhandle);
- }
-
- return rdd;
- }
-
- /**
- * TODO So far we only create broadcast variables but never destroy
- * them. This is a memory leak which might lead to executor out-of-memory.
- * However, in order to handle this, we need to keep track when broadcast
- * variables are no longer required.
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- public PartitionedBroadcastMatrix getBroadcastForVariable( String varname )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- MatrixObject mo = getMatrixObject(varname);
-
- PartitionedBroadcastMatrix bret = null;
-
- if( mo.getBroadcastHandle()!=null
- && mo.getBroadcastHandle().isValid() )
- {
- //reuse existing broadcast handle
- bret = mo.getBroadcastHandle().getBroadcast();
- }
- else
- {
- //obtain meta data for matrix
- int brlen = (int) mo.getNumRowsPerBlock();
- int bclen = (int) mo.getNumColumnsPerBlock();
-
- //create partitioned matrix block and release memory consumed by input
- MatrixBlock mb = mo.acquireRead();
- PartitionedMatrixBlock pmb = new PartitionedMatrixBlock(mb, brlen, bclen);
- mo.release();
-
- //determine coarse-grained partitioning
- int numPerPart = PartitionedBroadcastMatrix.computeBlocksPerPartition(mo.getNumRows(), mo.getNumColumns(), brlen, bclen);
- int numParts = (int) Math.ceil((double)pmb.getNumRowBlocks()*pmb.getNumColumnBlocks() / numPerPart);
- Broadcast<PartitionedMatrixBlock>[] ret = new Broadcast[numParts];
-
- //create coarse-grained partitioned broadcasts
- if( numParts > 1 ) {
- for( int i=0; i<numParts; i++ ) {
- int offset = i * numPerPart;
- int numBlks = Math.min(numPerPart, pmb.getNumRowBlocks()*pmb.getNumColumnBlocks()-offset);
- PartitionedMatrixBlock tmp = pmb.createPartition(offset, numBlks);
- ret[i] = getSparkContext().broadcast(tmp);
- }
- }
- else { //single partition
- ret[0] = getSparkContext().broadcast( pmb);
- }
-
- bret = new PartitionedBroadcastMatrix(ret);
- BroadcastObject bchandle = new BroadcastObject(bret, varname);
- mo.setBroadcastHandle(bchandle);
- }
-
- return bret;
- }
-
- /**
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public BlockPartitioner getPartitionerForRDDVariable(String varname)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //get input rdd and matrix characteristics
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = getBinaryBlockRDDHandleForVariable(varname);
- MatrixCharacteristics mc = getMatrixCharacteristics(varname);
-
- //create tile-based matrix partitioner
- return new BlockPartitioner(mc, in.partitions().size());
- }
-
- /**
- * Keep the output rdd of spark rdd operations as meta data of matrix objects in the
- * symbol table.
- *
- * Spark instructions should call this for all matrix outputs.
- *
- *
- * @param varname
- * @param rdd
- * @throws DMLRuntimeException
- */
- public void setRDDHandleForVariable(String varname, JavaPairRDD<MatrixIndexes,?> rdd)
- throws DMLRuntimeException
- {
- MatrixObject mo = getMatrixObject(varname);
- RDDObject rddhandle = new RDDObject(rdd, varname);
- mo.setRDDHandle( rddhandle );
- }
-
- /**
- * Utility method for creating an RDD out of an in-memory matrix block.
- *
- * @param sc
- * @param block
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- public static JavaPairRDD<MatrixIndexes,MatrixBlock> toJavaPairRDD(JavaSparkContext sc, MatrixBlock src, int brlen, int bclen)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- LinkedList<Tuple2<MatrixIndexes,MatrixBlock>> list = new LinkedList<Tuple2<MatrixIndexes,MatrixBlock>>();
-
- if( src.getNumRows() <= brlen
- && src.getNumColumns() <= bclen )
- {
- list.addLast(new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(1,1), src));
- }
- else
- {
- boolean sparse = src.isInSparseFormat();
-
- //create and write subblocks of matrix
- for(int blockRow = 0; blockRow < (int)Math.ceil(src.getNumRows()/(double)brlen); blockRow++)
- for(int blockCol = 0; blockCol < (int)Math.ceil(src.getNumColumns()/(double)bclen); blockCol++)
- {
- int maxRow = (blockRow*brlen + brlen < src.getNumRows()) ? brlen : src.getNumRows() - blockRow*brlen;
- int maxCol = (blockCol*bclen + bclen < src.getNumColumns()) ? bclen : src.getNumColumns() - blockCol*bclen;
-
- MatrixBlock block = new MatrixBlock(maxRow, maxCol, sparse);
-
- int row_offset = blockRow*brlen;
- int col_offset = blockCol*bclen;
-
- //copy submatrix to block
- src.sliceOperations( row_offset, row_offset+maxRow-1,
- col_offset, col_offset+maxCol-1, block );
-
- //append block to sequence file
- MatrixIndexes indexes = new MatrixIndexes(blockRow+1, blockCol+1);
- list.addLast(new Tuple2<MatrixIndexes,MatrixBlock>(indexes, block));
- }
- }
-
- return sc.parallelizePairs(list);
- }
-
- /**
- * This method is a generic abstraction for calls from the buffer pool.
- * See toMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int numRows, int numCols);
- *
- * @param rdd
- * @param numRows
- * @param numCols
- * @return
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("unchecked")
- public static MatrixBlock toMatrixBlock(RDDObject rdd, int rlen, int clen, int brlen, int bclen, long nnz)
- throws DMLRuntimeException
- {
- return toMatrixBlock(
- (JavaPairRDD<MatrixIndexes, MatrixBlock>) rdd.getRDD(),
- rlen, clen, brlen, bclen, nnz);
- }
-
- /**
- * Utility method for creating a single matrix block out of an RDD. Note that this collect call
- * might trigger execution of any pending transformations.
- *
- * NOTE: This is an unguarded utility function, which requires memory for both the output matrix
- * and its collected, blocked representation.
- *
- * @param rdd
- * @param numRows
- * @param numCols
- * @return
- * @throws DMLRuntimeException
- */
- public static MatrixBlock toMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int rlen, int clen, int brlen, int bclen, long nnz)
- throws DMLRuntimeException
- {
- MatrixBlock out = null;
-
- if( rlen <= brlen && clen <= bclen ) //SINGLE BLOCK
- {
- //special case without copy and nnz maintenance
- List<Tuple2<MatrixIndexes,MatrixBlock>> list = rdd.collect();
- if( list.size()>1 )
- throw new DMLRuntimeException("Expecting no more than one result block.");
- else if( list.size()==1 )
- out = list.get(0)._2();
- else //empty (e.g., after ops w/ outputEmpty=false)
- out = new MatrixBlock(rlen, clen, true);
- }
- else //MULTIPLE BLOCKS
- {
- //determine target sparse/dense representation
- long lnnz = (nnz >= 0) ? nnz : (long)rlen * clen;
- boolean sparse = MatrixBlock.evalSparseFormatInMemory(rlen, clen, lnnz);
-
- //create output matrix block (w/ lazy allocation)
- out = new MatrixBlock(rlen, clen, sparse);
- List<Tuple2<MatrixIndexes,MatrixBlock>> list = rdd.collect();
-
- //copy blocks one-at-a-time into output matrix block
- for( Tuple2<MatrixIndexes,MatrixBlock> keyval : list )
- {
- //unpack index-block pair
- MatrixIndexes ix = keyval._1();
- MatrixBlock block = keyval._2();
-
- //compute row/column block offsets
- int row_offset = (int)(ix.getRowIndex()-1)*brlen;
- int col_offset = (int)(ix.getColumnIndex()-1)*bclen;
- int rows = block.getNumRows();
- int cols = block.getNumColumns();
-
- if( sparse ) { //SPARSE OUTPUT
- //append block to sparse target in order to avoid shifting
- //note: this append requires a final sort of sparse rows
- out.appendToSparse(block, row_offset, col_offset);
- }
- else { //DENSE OUTPUT
- out.copy( row_offset, row_offset+rows-1,
- col_offset, col_offset+cols-1, block, false );
- }
- }
-
- //post-processing output matrix
- if( sparse )
- out.sortSparseRows();
- out.recomputeNonZeros();
- out.examSparsity();
- }
-
- return out;
- }
-
- /**
- *
- * @param rdd
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param nnz
- * @return
- * @throws DMLRuntimeException
- */
- public static PartitionedMatrixBlock toPartitionedMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int rlen, int clen, int brlen, int bclen, long nnz)
- throws DMLRuntimeException
- {
- PartitionedMatrixBlock out = new PartitionedMatrixBlock(rlen, clen, brlen, bclen);
-
- List<Tuple2<MatrixIndexes,MatrixBlock>> list = rdd.collect();
-
- //copy blocks one-at-a-time into output matrix block
- for( Tuple2<MatrixIndexes,MatrixBlock> keyval : list )
- {
- //unpack index-block pair
- MatrixIndexes ix = keyval._1();
- MatrixBlock block = keyval._2();
- out.setMatrixBlock((int)ix.getRowIndex(), (int)ix.getColumnIndex(), block);
- }
-
- return out;
- }
-
- /**
- *
- * @param rdd
- * @param oinfo
- */
- @SuppressWarnings("unchecked")
- public static long writeRDDtoHDFS( RDDObject rdd, String path, OutputInfo oinfo )
- {
- JavaPairRDD<MatrixIndexes,MatrixBlock> lrdd = (JavaPairRDD<MatrixIndexes, MatrixBlock>) rdd.getRDD();
-
- //recompute nnz
- long nnz = SparkUtils.computeNNZFromBlocks(lrdd);
-
- //save file is an action which also triggers nnz maintenance
- lrdd.saveAsHadoopFile(path,
- oinfo.outputKeyClass,
- oinfo.outputValueClass,
- oinfo.outputFormatClass);
-
- //return nnz aggregate of all blocks
- return nnz;
- }
-
-
- /**
- * Returns the available memory budget for broadcast variables in bytes.
- * In detail, this takes into account the total executor memory as well
- * as relative ratios for data and shuffle. Note, that this is a conservative
- * estimate since both data memory and shuffle memory might not be fully
- * utilized.
- *
- * @return
- */
- public static double getBroadcastMemoryBudget()
- {
- if( _memExecutors < 0 || _memRatioData < 0 || _memRatioShuffle < 0 )
- analyzeSparkConfiguation();
-
- //70% of remaining free memory
- double membudget = OptimizerUtils.MEM_UTIL_FACTOR *
- ( _memExecutors
- - _memExecutors*(_memRatioData+_memRatioShuffle) );
-
- return membudget;
- }
-
- /**
- *
- * @return
- */
- public static double getConfiguredTotalDataMemory() {
- return getConfiguredTotalDataMemory(false);
- }
-
- /**
- *
- * @param refresh
- * @return
- */
- public static double getConfiguredTotalDataMemory(boolean refresh)
- {
- if( _memExecutors < 0 || _memRatioData < 0 )
- analyzeSparkConfiguation();
-
- //always get the current num executors on refresh because this might
- //change if not all executors are initially allocated and it is plan-relevant
- if( refresh && !_confOnly ) {
- JavaSparkContext jsc = getSparkContextStatic();
- int numExec = Math.max(jsc.sc().getExecutorMemoryStatus().size() - 1, 1);
- return _memExecutors * _memRatioData * numExec;
- }
- else
- return ( _memExecutors * _memRatioData * _numExecutors );
- }
-
- public static int getNumExecutors()
- {
- if( _numExecutors < 0 )
- analyzeSparkConfiguation();
-
- return _numExecutors;
- }
-
- public static int getDefaultParallelism() {
- return getDefaultParallelism(false);
- }
-
- /**
- *
- * @return
- */
- public static int getDefaultParallelism(boolean refresh)
- {
- if( _defaultPar < 0 && !refresh )
- analyzeSparkConfiguation();
-
- //always get the current default parallelism on refresh because this might
- //change if not all executors are initially allocated and it is plan-relevant
- if( refresh && !_confOnly )
- return getSparkContextStatic().defaultParallelism();
- else
- return _defaultPar;
- }
-
- /**
- *
- */
- public static void analyzeSparkConfiguation()
- {
- SparkConf sconf = new SparkConf();
-
- //parse absolute executor memory
- String tmp = sconf.get("spark.executor.memory", "512m");
- if ( tmp.endsWith("g") || tmp.endsWith("G") )
- _memExecutors = Long.parseLong(tmp.substring(0,tmp.length()-1)) * 1024 * 1024 * 1024;
- else if ( tmp.endsWith("m") || tmp.endsWith("M") )
- _memExecutors = Long.parseLong(tmp.substring(0,tmp.length()-1)) * 1024 * 1024;
- else if( tmp.endsWith("k") || tmp.endsWith("K") )
- _memExecutors = Long.parseLong(tmp.substring(0,tmp.length()-1)) * 1024;
- else
- _memExecutors = Long.parseLong(tmp.substring(0,tmp.length()-2));
-
- //get data and shuffle memory ratios (defaults not specified in job conf)
- _memRatioData = sconf.getDouble("spark.storage.memoryFraction", 0.6); //default 60%
- _memRatioShuffle = sconf.getDouble("spark.shuffle.memoryFraction", 0.2); //default 20%
-
- int numExecutors = sconf.getInt("spark.executor.instances", -1);
- int numCoresPerExec = sconf.getInt("spark.executor.cores", -1);
- int defaultPar = sconf.getInt("spark.default.parallelism", -1);
-
- if( numExecutors > 1 && (defaultPar > 1 || numCoresPerExec > 1) ) {
- _numExecutors = numExecutors;
- _defaultPar = (defaultPar>1) ? defaultPar : numExecutors * numCoresPerExec;
- _confOnly = true;
- }
- else {
- //get default parallelism (total number of executors and cores)
- //note: spark context provides this information while conf does not
- //(for num executors we need to correct for driver and local mode)
- JavaSparkContext jsc = getSparkContextStatic();
- _numExecutors = Math.max(jsc.sc().getExecutorMemoryStatus().size() - 1, 1);
- _defaultPar = jsc.defaultParallelism();
- _confOnly = false; //implies env info refresh w/ spark context
- }
-
- //note: required time for infrastructure analysis on 5 node cluster: ~5-20ms.
- }
-
- /**
- *
- */
- public void checkAndRaiseValidationWarningJDKVersion()
- {
- //check for jdk version less than jdk8
- boolean isLtJDK8 = InfrastructureAnalyzer.isJavaVersionLessThanJDK8();
-
- //check multi-threaded executors
- int numExecutors = getNumExecutors();
- int numCores = getDefaultParallelism();
- boolean multiThreaded = (numCores > numExecutors);
-
- //check for jdk version less than 8 (and raise warning if multi-threaded)
- if( isLtJDK8 && multiThreaded)
- {
- //get the jre version
- String version = System.getProperty("java.version");
-
- LOG.warn("########################################################################################");
- LOG.warn("### WARNING: Multi-threaded text reblock may lead to thread contention on JRE < 1.8 ####");
- LOG.warn("### java.version = " + version);
- LOG.warn("### total number of executors = " + numExecutors);
- LOG.warn("### total number of cores = " + numCores);
- LOG.warn("### JDK-7032154: Performance tuning of sun.misc.FloatingDecimal/FormattedFloatingDecimal");
- LOG.warn("### Workaround: Convert text to binary w/ changed configuration of one executor per core");
- LOG.warn("########################################################################################");
- }
- }
-
- ///////////////////////////////////////////
- // Cleanup of RDDs and Broadcast variables
- ///////
-
- /**
- * Adds a child rdd object to the lineage of a parent rdd.
- *
- * @param varParent
- * @param varChild
- * @throws DMLRuntimeException
- */
- public void addLineageRDD(String varParent, String varChild)
- throws DMLRuntimeException
- {
- RDDObject parent = getMatrixObject(varParent).getRDDHandle();
- RDDObject child = getMatrixObject(varChild).getRDDHandle();
-
- parent.addLineageChild( child );
- }
-
- /**
- * Adds a child broadcast object to the lineage of a parent rdd.
- *
- * @param varParent
- * @param varChild
- * @throws DMLRuntimeException
- */
- public void addLineageBroadcast(String varParent, String varChild)
- throws DMLRuntimeException
- {
- RDDObject parent = getMatrixObject(varParent).getRDDHandle();
- BroadcastObject child = getMatrixObject(varChild).getBroadcastHandle();
-
- parent.addLineageChild( child );
- }
-
- @Override
- public void cleanupMatrixObject( MatrixObject mo )
- throws DMLRuntimeException
- {
- //NOTE: this method overwrites the default behavior of cleanupMatrixObject
- //and hence is transparently used by rmvar instructions and other users. The
- //core difference is the lineage-based cleanup of RDD and broadcast variables.
-
- try
- {
- if ( mo.isCleanupEnabled() )
- {
- //compute ref count only if matrix cleanup actually necessary
- if ( !getVariables().hasReferences(mo) )
- {
- //clean cached data
- mo.clearData();
-
- //clean hdfs data if no pending rdd operations on it
- if( mo.isFileExists() && mo.getFileName()!=null ) {
- if( mo.getRDDHandle()==null ) {
- MapReduceTool.deleteFileWithMTDIfExistOnHDFS(mo.getFileName());
- }
- else { //deferred file removal
- RDDObject rdd = mo.getRDDHandle();
- rdd.setHDFSFilename(mo.getFileName());
- }
- }
-
- //cleanup RDD and broadcast variables (recursive)
- //note: requires that mo.clearData already removed back references
- if( mo.getRDDHandle()!=null ) {
- rCleanupLineageObject(mo.getRDDHandle());
- }
- if( mo.getBroadcastHandle()!=null ) {
- rCleanupLineageObject(mo.getBroadcastHandle());
- }
- }
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
- /**
- *
- * @param lob
- * @throws IOException
- */
- private void rCleanupLineageObject(LineageObject lob)
- throws IOException
- {
- //abort recursive cleanup if still consumers
- if( lob.getNumReferences() > 0 )
- return;
-
- //abort if still reachable through matrix object (via back references for
- //robustness in function calls and to prevent repeated scans of the symbol table)
- if( lob.hasBackReference() )
- return;
-
- //cleanup current lineage object (from driver/executors)
- //incl deferred hdfs file removal (only if metadata set by cleanup call)
- if( lob instanceof RDDObject ) {
- RDDObject rdd = (RDDObject)lob;
- cleanupRDDVariable(rdd.getRDD());
- if( rdd.getHDFSFilename()!=null ) { //deferred file removal
- MapReduceTool.deleteFileWithMTDIfExistOnHDFS(rdd.getHDFSFilename());
- }
- }
- else if( lob instanceof BroadcastObject ) {
- PartitionedBroadcastMatrix pbm = ((BroadcastObject)lob).getBroadcast();
- for( Broadcast<PartitionedMatrixBlock> bc : pbm.getBroadcasts() )
- cleanupBroadcastVariable(bc);
- }
-
- //recursively process lineage children
- for( LineageObject c : lob.getLineageChilds() ){
- c.decrementNumReferences();
- rCleanupLineageObject(c);
- }
- }
-
- /**
- * This call destroys a broadcast variable at all executors and the driver.
- * Hence, it is intended to be used on rmvar only. Depending on the
- * ASYNCHRONOUS_VAR_DESTROY configuration, this is asynchronous or not.
- *
- *
- * @param inV
- */
- public void cleanupBroadcastVariable(Broadcast<?> bvar)
- {
- //in comparison to 'unpersist' (which would only delete the broadcast from the executors),
- //this call also deletes related data from the driver.
- if( bvar.isValid() ) {
- bvar.destroy( ASYNCHRONOUS_VAR_DESTROY );
- }
- }
-
- /**
- * This call removes an rdd variable from executor memory and disk if required.
- * Hence, it is intended to be used on rmvar only. Depending on the
- * ASYNCHRONOUS_VAR_DESTROY configuration, this is asynchronous or not.
- *
- * @param rvar
- */
- public void cleanupRDDVariable(JavaPairRDD<?,?> rvar)
- {
- if( rvar.getStorageLevel()!=StorageLevel.NONE() ) {
- rvar.unpersist( ASYNCHRONOUS_VAR_DESTROY );
- }
- }
-
- /**
- *
- * @param var
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- public void repartitionAndCacheMatrixObject( String var )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //get input rdd and default storage level
- MatrixObject mo = getMatrixObject(var);
- MatrixCharacteristics mcIn = mo.getMatrixCharacteristics();
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- getRDDHandleForMatrixObject(mo, InputInfo.BinaryBlockInputInfo);
-
- //avoid unnecessary caching of input in order to reduce memory pressure
- if( mo.getRDDHandle().allowsShortCircuitRead()
- && isRDDMarkedForCaching(in.id()) && !isRDDCached(in.id()) ) {
- in = (JavaPairRDD<MatrixIndexes,MatrixBlock>)
- ((RDDObject)mo.getRDDHandle().getLineageChilds().get(0)).getRDD();
-
- //investigate issue of unnecessarily large number of partitions
- int numPartitions = CheckpointSPInstruction.getNumCoalescePartitions(mcIn, in);
- if( numPartitions < in.partitions().size() )
- in = in.coalesce( numPartitions );
- }
-
- //repartition and persist rdd (force creation of shuffled rdd via merge)
- JavaPairRDD<MatrixIndexes,MatrixBlock> out = RDDAggregateUtils.mergeByKey(in);
- out.persist( Checkpoint.DEFAULT_STORAGE_LEVEL )
- .count(); //trigger caching to prevent contention
-
- //create new rdd handle, in-place of current matrix object
- RDDObject inro = mo.getRDDHandle(); //guaranteed to exist (see above)
- RDDObject outro = new RDDObject(out, var); //create new rdd object
- outro.setCheckpointRDD(true); //mark as checkpointed
- outro.addLineageChild(inro); //keep lineage to prevent cycles on cleanup
- mo.setRDDHandle(outro);
- }
-
- /**
- *
- * @param var
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unchecked")
- public void cacheMatrixObject( String var )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //get input rdd and default storage level
- MatrixObject mo = getMatrixObject(var);
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- getRDDHandleForMatrixObject(mo, InputInfo.BinaryBlockInputInfo);
-
- //persist rdd (force rdd caching)
- in.count(); //trigger caching to prevent contention
- }
-
- /**
- *
- * @param poolName
- */
- public void setThreadLocalSchedulerPool(String poolName) {
- if( FAIR_SCHEDULER_MODE ) {
- getSparkContext().sc().setLocalProperty(
- "spark.scheduler.pool", poolName);
- }
- }
-
- /**
- *
- */
- public void cleanupThreadLocalSchedulerPool() {
- if( FAIR_SCHEDULER_MODE ) {
- getSparkContext().sc().setLocalProperty(
- "spark.scheduler.pool", null);
- }
- }
-
- /**
- *
- * @param rddID
- * @return
- */
- private boolean isRDDMarkedForCaching( int rddID ) {
- JavaSparkContext jsc = getSparkContext();
- return jsc.sc().getPersistentRDDs().contains(rddID);
- }
-
- /**
- *
- * @param rddID
- * @return
- */
- private boolean isRDDCached( int rddID ) {
- //check that rdd is marked for caching
- JavaSparkContext jsc = getSparkContext();
- if( !jsc.sc().getPersistentRDDs().contains(rddID) ) {
- return false;
- }
-
- //check that rdd is actually already cached
- for( RDDInfo info : jsc.sc().getRDDStorageInfo() ) {
- if( info.id() == rddID )
- return info.isCached();
- }
- return false;
- }
-
- ///////////////////////////////////////////
- // Debug String Handling (see explain); TODO to be removed
- ///////
-
- /**
- *
- * @param inst
- * @param outputVarName
- * @throws DMLRuntimeException
- */
- public void setDebugString(SPInstruction inst, String outputVarName)
- throws DMLRuntimeException
- {
- RDDObject parentLineage = getMatrixObject(outputVarName).getRDDHandle();
-
- if( parentLineage == null || parentLineage.getRDD() == null )
- return;
-
- MLContextProxy.addRDDForInstructionForMonitoring(inst, parentLineage.getRDD().id());
-
- JavaPairRDD<?, ?> out = parentLineage.getRDD();
- JavaPairRDD<?, ?> in1 = null;
- JavaPairRDD<?, ?> in2 = null;
- String input1VarName = null;
- String input2VarName = null;
- if(parentLineage.getLineageChilds() != null) {
- for(LineageObject child : parentLineage.getLineageChilds()) {
- if(child instanceof RDDObject) {
- if(in1 == null) {
- in1 = ((RDDObject) child).getRDD();
- input1VarName = child.getVarName();
- }
- else if(in2 == null) {
- in2 = ((RDDObject) child).getRDD();
- input2VarName = child.getVarName();
- }
- else {
- throw new DMLRuntimeException("PRINT_EXPLAIN_WITH_LINEAGE not yet supported for three outputs");
- }
- }
- }
- }
- setLineageInfoForExplain(inst, out, in1, input1VarName, in2, input2VarName);
- }
-
- // The most expensive operation here is rdd.toDebugString() which can be a major hit because
- // of unrolling lazy evaluation of Spark. Hence, it is guarded against it along with flag 'PRINT_EXPLAIN_WITH_LINEAGE' which is
- // enabled only through MLContext. This way, it doesnot affect our performance evaluation through non-MLContext path
- private void setLineageInfoForExplain(SPInstruction inst,
- JavaPairRDD<?, ?> out,
- JavaPairRDD<?, ?> in1, String in1Name,
- JavaPairRDD<?, ?> in2, String in2Name) throws DMLRuntimeException {
-
-
- // RDDInfo outInfo = org.apache.spark.storage.RDDInfo.fromRdd(out.rdd());
-
- // First fetch start lines from input RDDs
- String startLine1 = null;
- String startLine2 = null;
- int i1length = 0, i2length = 0;
- if(in1 != null) {
- String [] lines = in1.toDebugString().split("\\r?\\n");
- startLine1 = SparkUtils.getStartLineFromSparkDebugInfo(lines[0]); // lines[0].substring(4, lines[0].length());
- i1length = lines.length;
- }
- if(in2 != null) {
- String [] lines = in2.toDebugString().split("\\r?\\n");
- startLine2 = SparkUtils.getStartLineFromSparkDebugInfo(lines[0]); // lines[0].substring(4, lines[0].length());
- i2length = lines.length;
- }
-
- String outDebugString = "";
- int skip = 0;
-
- // Now process output RDD and replace inputRDD debug string by the matrix variable name
- String [] outLines = out.toDebugString().split("\\r?\\n");
- for(int i = 0; i < outLines.length; i++) {
- if(skip > 0) {
- skip--;
- // outDebugString += "\nSKIP:" + outLines[i];
- }
- else if(startLine1 != null && outLines[i].contains(startLine1)) {
- String prefix = SparkUtils.getPrefixFromSparkDebugInfo(outLines[i]); // outLines[i].substring(0, outLines[i].length() - startLine1.length());
- outDebugString += "\n" + prefix + "[[" + in1Name + "]]";
- //outDebugString += "\n{" + prefix + "}[[" + in1Name + "]] => " + outLines[i];
- skip = i1length - 1;
- }
- else if(startLine2 != null && outLines[i].contains(startLine2)) {
- String prefix = SparkUtils.getPrefixFromSparkDebugInfo(outLines[i]); // outLines[i].substring(0, outLines[i].length() - startLine2.length());
- outDebugString += "\n" + prefix + "[[" + in2Name + "]]";
- skip = i2length - 1;
- }
- else {
- outDebugString += "\n" + outLines[i];
- }
- }
-
- MLContext mlContext = MLContextProxy.getActiveMLContext();
- if(mlContext != null && mlContext.getMonitoringUtil() != null) {
- mlContext.getMonitoringUtil().setLineageInfo(inst, outDebugString);
- }
- else {
- throw new DMLRuntimeException("The method setLineageInfoForExplain should be called only through MLContext");
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitioner.java
deleted file mode 100644
index a5c05e0..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitioner.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-
-
-/**
- * This is the base class for all data partitioner.
- *
- */
-public abstract class DataPartitioner
-{
-
- protected static final Log LOG = LogFactory.getLog(DataPartitioner.class.getName());
-
- //note: the following value has been empirically determined but might change in the future,
- //MatrixBlockDSM.SPARCITY_TURN_POINT (with 0.4) was too high because we create 3-4 values per nnz and
- //have some computation overhead for binary cell.
- protected static final double SPARSITY_CELL_THRESHOLD = 0.1d;
-
- protected static final String NAME_SUFFIX = "_dp";
-
- //instance variables
- protected PDataPartitionFormat _format = null;
- protected int _n = -1; //blocksize if applicable
- protected boolean _allowBinarycell = true;
-
- protected DataPartitioner( PDataPartitionFormat dpf, int n )
- {
- _format = dpf;
- _n = n;
- }
-
-
-
- /**
- *
- * @param in
- * @param fnameNew
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixObject createPartitionedMatrixObject( MatrixObject in, String fnameNew )
- throws DMLRuntimeException
- {
- return createPartitionedMatrixObject(in, fnameNew, false);
- }
-
- /**
- *
- * @param in
- * @param fnameNew
- * @param force
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixObject createPartitionedMatrixObject( MatrixObject in, String fnameNew, boolean force )
- throws DMLRuntimeException
- {
- ValueType vt = in.getValueType();
- String varname = in.getVarName();
- MatrixObject out = new MatrixObject(vt, fnameNew );
- out.setDataType( DataType.MATRIX );
- out.setVarName( varname+NAME_SUFFIX );
-
- return createPartitionedMatrixObject(in, out, force);
- }
-
-
- /**
- * Creates a partitioned matrix object based on the given input matrix object,
- * according to the specified split format. The input matrix can be in-memory
- * or still on HDFS and the partitioned output matrix is written to HDFS. The
- * created matrix object can be used transparently for obtaining the full matrix
- * or reading 1 or multiple partitions based on given index ranges.
- *
- * @param in
- * @param force
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixObject createPartitionedMatrixObject( MatrixObject in, MatrixObject out, boolean force )
- throws DMLRuntimeException
- {
- //check for naive partitioning
- if( _format == PDataPartitionFormat.NONE )
- return in;
-
- //analyze input matrix object
- MatrixFormatMetaData meta = (MatrixFormatMetaData)in.getMetaData();
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- InputInfo ii = meta.getInputInfo();
- OutputInfo oi = meta.getOutputInfo();
- long rows = mc.getRows();
- long cols = mc.getCols();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
- long nonZeros = mc.getNonZeros();
- double sparsity = (nonZeros>=0 && rows>0 && cols>0)?
- ((double)nonZeros)/(rows*cols) : 1.0;
-
- if( !force ) //try to optimize, if format not forced
- {
- //check lower bound of useful data partitioning
- if( rows < Hop.CPThreshold && cols < Hop.CPThreshold ) //or matrix already fits in mem
- {
- return in;
- }
-
- //check for changing to blockwise representations
- if( _format == PDataPartitionFormat.ROW_WISE && cols < Hop.CPThreshold )
- {
- LOG.debug("Changing format from "+PDataPartitionFormat.ROW_WISE+" to "+PDataPartitionFormat.ROW_BLOCK_WISE+".");
- _format = PDataPartitionFormat.ROW_BLOCK_WISE;
- }
- if( _format == PDataPartitionFormat.COLUMN_WISE && rows < Hop.CPThreshold )
- {
- LOG.debug("Changing format from "+PDataPartitionFormat.COLUMN_WISE+" to "+PDataPartitionFormat.ROW_BLOCK_WISE+".");
- _format = PDataPartitionFormat.COLUMN_BLOCK_WISE;
- }
- //_format = PDataPartitionFormat.ROW_BLOCK_WISE_N;
- }
-
- //check changing to binarycell in case of sparse cols (robustness)
- boolean convertBlock2Cell = false;
- if( ii == InputInfo.BinaryBlockInputInfo
- && _allowBinarycell
- && _format == PDataPartitionFormat.COLUMN_WISE
- && sparsity < SPARSITY_CELL_THRESHOLD )
- {
- LOG.debug("Changing partition outputinfo from binaryblock to binarycell due to sparsity="+sparsity);
- oi = OutputInfo.BinaryCellOutputInfo;
- convertBlock2Cell = true;
- }
-
- //prepare filenames and cleanup if required
- String fnameNew = out.getFileName();
- try{
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
- }
- catch(Exception ex){
- throw new DMLRuntimeException( ex );
- }
-
- //core partitioning (depending on subclass)
- partitionMatrix( in, fnameNew, ii, oi, rows, cols, brlen, bclen );
-
- //create output matrix object
- out.setPartitioned( _format, _n );
-
- MatrixCharacteristics mcNew = new MatrixCharacteristics( rows, cols, (int)brlen, (int)bclen );
- mcNew.setNonZeros( nonZeros );
- if( convertBlock2Cell )
- ii = InputInfo.BinaryCellInputInfo;
- MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,oi,ii);
- out.setMetaData(metaNew);
-
- return out;
-
- }
-
- /**
- *
- */
- public void disableBinaryCell()
- {
- _allowBinarycell = false;
- }
-
- /**
- *
- * @param fname
- * @param fnameNew
- * @param ii
- * @param oi
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- protected abstract void partitionMatrix( MatrixObject in, String fnameNew, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen )
- throws DMLRuntimeException;
-
-
- public static MatrixBlock createReuseMatrixBlock( PDataPartitionFormat dpf, int rows, int cols )
- {
- MatrixBlock tmp = null;
-
- switch( dpf )
- {
- case ROW_WISE:
- //default assumption sparse, but reset per input block anyway
- tmp = new MatrixBlock( 1, (int)cols, true, (int)(cols*0.1) );
- break;
- case COLUMN_WISE:
- //default dense because single column alwyas below SKINNY_MATRIX_TURN_POINT
- tmp = new MatrixBlock( (int)rows, 1, false );
- break;
- default:
- //do nothing
- }
-
- return tmp;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerLocal.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerLocal.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerLocal.java
deleted file mode 100644
index b3a5075..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerLocal.java
+++ /dev/null
@@ -1,888 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.Cell;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-/**
- * Partitions a given matrix into row or column partitions with a two pass-approach.
- * In the first phase the input matrix is read from HDFS and sorted into block partitions
- * in a staging area in the local file system according to the partition format.
- * In order to allow for scalable partitioning, we process one block at a time.
- * Furthermore, in the second phase, all blocks of a partition are append to a sequence file
- * on HDFS. Block-wise partitioning and write-once semantics of sequence files require the
- * indirection over the local staging area. For scalable computation, we process one
- * sequence file at a time.
- *
- * NOTE: For the resulting partitioned matrix, we store block and cell indexes wrt partition boundaries.
- * This means that the partitioned matrix CANNOT be read as a traditional matrix because there are
- * for example multiple blocks with same index (while the actual index is encoded in the path).
- * In order to enable full read of partition matrices, data converter would need to handle negative
- * row/col offsets for partitioned read. Currently not done in order to avoid overhead from normal read
- * and since partitioning only applied if exclusively indexed access.
- *
- *
- */
-public class DataPartitionerLocal extends DataPartitioner
-{
-
- private static final boolean PARALLEL = true;
-
- private IDSequence _seq = null;
- private MatrixBlock _reuseBlk = null;
-
- private int _par = -1;
-
- /**
- *
- * @param dpf
- * @param n
- * @param par -1 for serial otherwise number of threads, can be ignored by implementation
- * @throws DMLRuntimeException
- */
- public DataPartitionerLocal(PDataPartitionFormat dpf, int n, int par)
- throws DMLRuntimeException
- {
- super(dpf, n);
-
- //TODO
- if( dpf == PDataPartitionFormat.ROW_BLOCK_WISE_N || dpf == PDataPartitionFormat.COLUMN_BLOCK_WISE_N )
- throw new DMLRuntimeException("Data partitioning formt '"+dpf+"' not supported by DataPartitionerLocal" );
-
- _seq = new IDSequence();
- _par = (par > 0) ? par : 1;
- }
-
- @Override
- protected void partitionMatrix(MatrixObject in, String fnameNew, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen)
- throws DMLRuntimeException
- {
- //force writing to disk (typically not required since partitioning only applied if dataset exceeds CP size)
- in.exportData(); //written to disk iff dirty
-
- String fname = in.getFileName();
- String fnameStaging = LocalFileUtils.getUniqueWorkingDir( LocalFileUtils.CATEGORY_PARTITIONING );
-
- //reblock input matrix
- if( ii == InputInfo.TextCellInputInfo )
- partitionTextCell( fname, fnameStaging, fnameNew, rlen, clen, brlen, bclen );
- else if( ii == InputInfo.BinaryCellInputInfo )
- partitionBinaryCell( fname, fnameStaging, fnameNew, rlen, clen, brlen, bclen );
- else if( ii == InputInfo.BinaryBlockInputInfo )
- {
- if( oi == OutputInfo.BinaryBlockOutputInfo )
- partitionBinaryBlock( fname, fnameStaging, fnameNew, rlen, clen, brlen, bclen );
- else if ( oi == OutputInfo.BinaryCellOutputInfo )
- partitionBinaryBlock2BinaryCell( fname, fnameStaging, fnameNew, rlen, clen, brlen, bclen );
- }
- else
- throw new DMLRuntimeException("Cannot create data partitions of format: "+ii.toString());
-
- LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
- }
-
-
-
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- private void partitionTextCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
- throws DMLRuntimeException
- {
- long row = -1;
- long col = -1;
-
- try
- {
- //STEP 1: read matrix from HDFS and write blocks to local staging area
- //check and add input path
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(fname);
- FileInputFormat.addInputPath(job, path);
- TextInputFormat informat = new TextInputFormat();
- informat.configure(job);
- InputSplit[] splits = informat.getSplits(job, 1);
-
- LinkedList<Cell> buffer = new LinkedList<Cell>();
- LongWritable key = new LongWritable();
- Text value = new Text();
- FastStringTokenizer st = new FastStringTokenizer(' ');
-
- for(InputSplit split: splits)
- {
- RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
- try
- {
- while(reader.next(key, value))
- {
- st.reset( value.toString() ); //reset tokenizer
- row = st.nextLong();
- col = st.nextLong();
- double lvalue = st.nextDouble();
- Cell tmp = new Cell( row, col, lvalue );
-
- buffer.addLast( tmp );
- if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE ) //periodic flush
- {
- appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
- buffer.clear();
- }
- }
-
- //final flush
- if( !buffer.isEmpty() )
- {
- appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
- buffer.clear();
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- //STEP 2: read matrix blocks from staging area and write matrix to HDFS
- String[] fnamesPartitions = new File(fnameStaging).list();
- if(PARALLEL)
- {
- int len = Math.min(fnamesPartitions.length, _par);
- Thread[] threads = new Thread[len];
- for( int i=0;i<len;i++ )
- {
- int start = i*(int)Math.ceil(((double)fnamesPartitions.length)/len);
- int end = (i+1)*(int)Math.ceil(((double)fnamesPartitions.length)/len)-1;
- end = Math.min(end, fnamesPartitions.length-1);
- threads[i] = new Thread(new DataPartitionerWorkerTextCell(job, fnameNew, fnameStaging, fnamesPartitions, start, end));
- threads[i].start();
- }
-
- for( Thread t : threads )
- t.join();
- }
- else
- {
- for( String pdir : fnamesPartitions )
- writeTextCellFileToHDFS( job, fnameNew, fnameStaging+"/"+pdir );
- }
- }
- catch (Exception e)
- {
- //post-mortem error handling and bounds checking
- if( row < 1 || row > rlen || col < 1 || col > clen )
- {
- throw new DMLRuntimeException("Matrix cell ["+(row)+","+(col)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
- }
- else
- throw new DMLRuntimeException("Unable to partition text cell matrix.", e);
- }
- }
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void partitionBinaryCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
- throws DMLRuntimeException
- {
- long row = -1;
- long col = -1;
-
- try
- {
- //STEP 1: read matrix from HDFS and write blocks to local staging area
- //check and add input path
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(fname);
- FileSystem fs = FileSystem.get(job);
-
- //prepare sequence file reader, and write to local staging area
- LinkedList<Cell> buffer = new LinkedList<Cell>();
- MatrixIndexes key = new MatrixIndexes();
- MatrixCell value = new MatrixCell();
-
- for( Path lpath : MatrixReader.getSequenceFilePaths(fs, path) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
- try
- {
- while(reader.next(key, value))
- {
- row = key.getRowIndex();
- col = key.getColumnIndex();
- Cell tmp = new Cell( row, col, value.getValue() );
-
- buffer.addLast( tmp );
- if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE ) //periodic flush
- {
- appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
- buffer.clear();
- }
- }
-
- //final flush
- if( !buffer.isEmpty() )
- {
- appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
- buffer.clear();
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- //STEP 2: read matrix blocks from staging area and write matrix to HDFS
- String[] fnamesPartitions = new File(fnameStaging).list();
- if(PARALLEL)
- {
- int len = Math.min(fnamesPartitions.length, _par);
- Thread[] threads = new Thread[len];
- for( int i=0;i<len;i++ )
- {
- int start = i*(int)Math.ceil(((double)fnamesPartitions.length)/len);
- int end = (i+1)*(int)Math.ceil(((double)fnamesPartitions.length)/len)-1;
- end = Math.min(end, fnamesPartitions.length-1);
- threads[i] = new Thread(new DataPartitionerWorkerBinaryCell(job, fnameNew, fnameStaging, fnamesPartitions, start, end));
- threads[i].start();
- }
-
- for( Thread t : threads )
- t.join();
- }
- else
- {
- for( String pdir : fnamesPartitions )
- writeBinaryCellSequenceFileToHDFS( job, fnameNew, fnameStaging+"/"+pdir );
- }
- }
- catch (Exception e)
- {
- //post-mortem error handling and bounds checking
- if( row < 1 || row > rlen || col < 1 || col > clen )
- {
- throw new DMLRuntimeException("Matrix cell ["+(row)+","+(col)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
- }
- else
- throw new DMLRuntimeException("Unable to partition binary cell matrix.", e);
- }
- }
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void partitionBinaryBlock( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
- throws DMLRuntimeException
- {
- try
- {
- //create reuse object
- _reuseBlk = DataPartitioner.createReuseMatrixBlock(_format, brlen, bclen);
-
- //STEP 1: read matrix from HDFS and write blocks to local staging area
- //check and add input path
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(fname);
- FileSystem fs = FileSystem.get(job);
-
- //prepare sequence file reader, and write to local staging area
- MatrixIndexes key = new MatrixIndexes();
- MatrixBlock value = new MatrixBlock();
-
- for(Path lpath : MatrixReader.getSequenceFilePaths(fs, path) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
- try
- {
- while(reader.next(key, value)) //for each block
- {
- long row_offset = (key.getRowIndex()-1)*brlen;
- long col_offset = (key.getColumnIndex()-1)*bclen;
- long rows = value.getNumRows();
- long cols = value.getNumColumns();
-
- //bound check per block
- if( row_offset + rows < 1 || row_offset + rows > rlen || col_offset + cols<1 || col_offset + cols > clen )
- {
- throw new IOException("Matrix block ["+(row_offset+1)+":"+(row_offset+rows)+","+(col_offset+1)+":"+(col_offset+cols)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
- }
-
- appendBlockToStagingArea(fnameStaging, value, row_offset, col_offset, brlen, bclen);
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- //STEP 2: read matrix blocks from staging area and write matrix to HDFS
- String[] fnamesPartitions = new File(fnameStaging).list();
- if(PARALLEL)
- {
- int len = Math.min(fnamesPartitions.length, _par);
- Thread[] threads = new Thread[len];
- for( int i=0;i<len;i++ )
- {
- int start = i*(int)Math.ceil(((double)fnamesPartitions.length)/len);
- int end = (i+1)*(int)Math.ceil(((double)fnamesPartitions.length)/len)-1;
- end = Math.min(end, fnamesPartitions.length-1);
- threads[i] = new Thread(new DataPartitionerWorkerBinaryBlock(job, fnameNew, fnameStaging, fnamesPartitions, start, end));
- threads[i].start();
- }
-
- for( Thread t : threads )
- t.join();
- }
- else
- {
- for( String pdir : fnamesPartitions )
- writeBinaryBlockSequenceFileToHDFS( job, fnameNew, fnameStaging+"/"+pdir, false );
- }
- }
- catch (Exception e)
- {
- throw new DMLRuntimeException("Unable to partition binary block matrix.", e);
- }
- }
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void partitionBinaryBlock2BinaryCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
- throws DMLRuntimeException
- {
- try
- {
- //STEP 1: read matrix from HDFS and write blocks to local staging area
- //check and add input path
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(fname);
- FileSystem fs = FileSystem.get(job);
-
- //prepare sequence file reader, and write to local staging area
- MatrixIndexes key = new MatrixIndexes();
- MatrixBlock value = new MatrixBlock();
-
- LinkedList<Cell> buffer = new LinkedList<Cell>();
-
- for(Path lpath : MatrixReader.getSequenceFilePaths(fs, path) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
- try
- {
- while(reader.next(key, value)) //for each block
- {
- long row_offset = (key.getRowIndex()-1)*brlen;
- long col_offset = (key.getColumnIndex()-1)*bclen;
- long rows = value.getNumRows();
- long cols = value.getNumColumns();
-
- //bound check per block
- if( row_offset + rows < 1 || row_offset + rows > rlen || col_offset + cols<1 || col_offset + cols > clen )
- {
- throw new IOException("Matrix block ["+(row_offset+1)+":"+(row_offset+rows)+","+(col_offset+1)+":"+(col_offset+cols)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
- }
-
- boolean sparse = value.isInSparseFormat();
- if( sparse ) //SPARSE
- {
- SparseRowsIterator iter = value.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- Cell tmp = new Cell( row_offset + lcell.i + 1,
- col_offset + lcell.j + 1,
- lcell.v );
- buffer.addLast( tmp );
- }
- }
- else //DENSE
- {
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- {
- double lvalue = value.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //for nnz
- {
- Cell tmp = new Cell( row_offset + i + 1,
- col_offset + j + 1,
- lvalue );
- buffer.addLast( tmp );
- }
- }
- }
-
- appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
- buffer.clear();
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
-
- //STEP 2: read matrix blocks from staging area and write matrix to HDFS
- String[] fnamesPartitions = new File(fnameStaging).list();
- if(PARALLEL)
- {
- int len = Math.min(fnamesPartitions.length, _par);
- Thread[] threads = new Thread[len];
- for( int i=0;i<len;i++ )
- {
- int start = i*(int)Math.ceil(((double)fnamesPartitions.length)/len);
- int end = (i+1)*(int)Math.ceil(((double)fnamesPartitions.length)/len)-1;
- end = Math.min(end, fnamesPartitions.length-1);
- threads[i] = new Thread(new DataPartitionerWorkerBinaryCell(job, fnameNew, fnameStaging, fnamesPartitions, start, end));
- threads[i].start();
- }
-
- for( Thread t : threads )
- t.join();
- }
- else
- {
- for( String pdir : fnamesPartitions )
- writeBinaryCellSequenceFileToHDFS( job, fnameNew, fnameStaging+"/"+pdir );
- }
- }
- catch (Exception e)
- {
- throw new DMLRuntimeException("Unable to partition binary block matrix.", e);
- }
- }
-
-
- /**
- *
- * @param dir
- * @param mb
- * @param row_offset
- * @param col_offset
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- * @throws DMLUnsupportedOperationException
- */
- private void appendBlockToStagingArea( String dir, MatrixBlock mb, long row_offset, long col_offset, long brlen, long bclen )
- throws DMLRuntimeException, IOException, DMLUnsupportedOperationException
- {
- //NOTE: for temporary block we always create dense representations
- boolean sparse = mb.isInSparseFormat();
- long nnz = mb.getNonZeros();
- long rows = mb.getNumRows();
- long cols = mb.getNumColumns();
- double sparsity = ((double)nnz)/(rows*cols);
-
- if( _format == PDataPartitionFormat.ROW_WISE )
- {
- _reuseBlk.reset( 1, (int)cols, sparse, (int) (cols*sparsity) );
- for( int i=0; i<rows; i++ )
- {
- String pdir = LocalFileUtils.checkAndCreateStagingDir(dir+"/"+(row_offset+1+i));
- String pfname = pdir+"/"+"block_"+(col_offset/bclen+1);
- mb.sliceOperations(i, i, 0, (int)(cols-1), _reuseBlk);
- LocalFileUtils.writeMatrixBlockToLocal(pfname, _reuseBlk);
- _reuseBlk.reset();
- }
- }
- else if( _format == PDataPartitionFormat.ROW_BLOCK_WISE )
- {
- String pdir = LocalFileUtils.checkAndCreateStagingDir(dir+"/"+(row_offset/brlen+1));
- String pfname = pdir+"/"+"block_"+(col_offset/bclen+1);
- LocalFileUtils.writeMatrixBlockToLocal(pfname, mb);
- }
- else if( _format == PDataPartitionFormat.COLUMN_WISE )
- {
- //create object for reuse
- _reuseBlk.reset( (int)rows, 1, false );
-
- for( int i=0; i<cols; i++ )
- {
- String pdir = LocalFileUtils.checkAndCreateStagingDir(dir+"/"+(col_offset+1+i));
- String pfname = pdir+"/"+"block_"+(row_offset/brlen+1);
- mb.sliceOperations(0, (int)(rows-1), i, i, _reuseBlk);
- LocalFileUtils.writeMatrixBlockToLocal(pfname, _reuseBlk);
- _reuseBlk.reset();
- }
- }
- else if( _format == PDataPartitionFormat.COLUMN_BLOCK_WISE )
- {
- String pdir = LocalFileUtils.checkAndCreateStagingDir(dir+"/"+(col_offset/bclen+1));
- String pfname = pdir+"/"+"block_"+(row_offset/brlen+1);
- LocalFileUtils.writeMatrixBlockToLocal(pfname, mb);
- }
- }
-
- /**
- *
- * @param dir
- * @param buffer
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void appendCellBufferToStagingArea( String dir, LinkedList<Cell> buffer, int brlen, int bclen )
- throws DMLRuntimeException, IOException
- {
- HashMap<Long,LinkedList<Cell>> sortedBuffer = new HashMap<Long,LinkedList<Cell>>();
-
- //sort cells in buffer wrt key
- long key = -1;
- for( Cell c : buffer )
- {
- switch(_format)
- {
- case ROW_WISE:
- key = c.getRow();
- c.setRow(1);
- break;
- case ROW_BLOCK_WISE:
- key = (c.getRow()-1)/brlen+1;
- c.setRow((c.getRow()-1)%brlen+1);
- break;
- case COLUMN_WISE:
- key = c.getCol();
- c.setCol(1);
- break;
- case COLUMN_BLOCK_WISE:
- key = (c.getCol()-1)/bclen+1;
- c.setCol((c.getCol()-1)%bclen+1);
- break;
- default:
- //do nothing
- }
-
- if( !sortedBuffer.containsKey(key) )
- sortedBuffer.put(key, new LinkedList<Cell>());
- sortedBuffer.get(key).addLast(c);
- }
-
- //write lists of cells to local files
- for( Entry<Long,LinkedList<Cell>> e : sortedBuffer.entrySet() )
- {
- String pdir = LocalFileUtils.checkAndCreateStagingDir(dir+"/"+e.getKey());
- String pfname = pdir+"/"+"block_"+_seq.getNextID();
- StagingFileUtils.writeCellListToLocal(pfname, e.getValue());
- }
- }
-
-
- /////////////////////////////////////
- // Helper methods for HDFS //
- // read/write in different formats //
- /////////////////////////////////////
-
- @SuppressWarnings("deprecation")
- public void writeBinaryBlockSequenceFileToHDFS( JobConf job, String dir, String lpdir, boolean threadsafe )
- throws IOException
- {
- long key = getKeyFromFilePath(lpdir);
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(dir+"/"+key);
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixBlock.class); //beware ca 50ms
-
- try
- {
- String[] fnameBlocks = new File( lpdir ).list();
- for( String fnameBlock : fnameBlocks )
- {
- long key2 = getKey2FromFileName(fnameBlock);
- MatrixBlock tmp = null;
- if( threadsafe )
- tmp = LocalFileUtils.readMatrixBlockFromLocal(lpdir+"/"+fnameBlock);
- else
- tmp = LocalFileUtils.readMatrixBlockFromLocal(lpdir+"/"+fnameBlock, _reuseBlk);
-
- if( _format == PDataPartitionFormat.ROW_WISE || _format == PDataPartitionFormat.ROW_BLOCK_WISE )
- {
- writer.append(new MatrixIndexes(1,key2), tmp);
- }
- else if( _format == PDataPartitionFormat.COLUMN_WISE || _format == PDataPartitionFormat.COLUMN_BLOCK_WISE )
- {
- writer.append(new MatrixIndexes(key2,1), tmp);
- }
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- @SuppressWarnings("deprecation")
- public void writeBinaryCellSequenceFileToHDFS( JobConf job, String dir, String lpdir )
- throws IOException
- {
- long key = getKeyFromFilePath(lpdir);
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(dir+"/"+key);
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class); //beware ca 50ms
-
- try
- {
- MatrixIndexes indexes = new MatrixIndexes();
- MatrixCell cell = new MatrixCell();
-
- String[] fnameBlocks = new File( lpdir ).list();
- for( String fnameBlock : fnameBlocks )
- {
- LinkedList<Cell> tmp = StagingFileUtils.readCellListFromLocal(lpdir+"/"+fnameBlock);
- for( Cell c : tmp )
- {
- indexes.setIndexes(c.getRow(), c.getCol());
- cell.setValue(c.getValue());
- writer.append(indexes, cell);
- }
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- public void writeTextCellFileToHDFS( JobConf job, String dir, String lpdir )
- throws IOException
- {
- long key = getKeyFromFilePath(lpdir);
- FileSystem fs = FileSystem.get(job);
- Path path = new Path(dir+"/"+key);
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
- try
- {
- //for obj reuse and preventing repeated buffer re-allocations
- StringBuilder sb = new StringBuilder();
-
- String[] fnameBlocks = new File( lpdir ).list();
- for( String fnameBlock : fnameBlocks )
- {
- LinkedList<Cell> tmp = StagingFileUtils.readCellListFromLocal(lpdir+"/"+fnameBlock);
- for( Cell c : tmp )
- {
- sb.append(c.getRow());
- sb.append(' ');
- sb.append(c.getCol());
- sb.append(' ');
- sb.append(c.getValue());
- sb.append('\n');
- out.write( sb.toString() );
- sb.setLength(0);
- }
- }
- }
- finally
- {
- if( out != null )
- out.close();
- }
- }
-
-
- /////////////////////////////////
- // Helper methods for local fs //
- // read/write //
- /////////////////////////////////
-
- /**
- *
- * @param dir
- * @return
- */
- private long getKeyFromFilePath( String dir )
- {
- String[] dirparts = dir.split("/");
- long key = Long.parseLong( dirparts[dirparts.length-1] );
- return key;
- }
-
- /**
- *
- * @param fname
- * @return
- */
- private long getKey2FromFileName( String fname )
- {
- return Long.parseLong( fname.split("_")[1] );
- }
-
- private abstract class DataPartitionerWorker implements Runnable
- {
- private JobConf _job = null;
- private String _fnameNew = null;
- private String _fnameStaging = null;
- private String[] _fnamesPartitions = null;
- private int _start = -1;
- private int _end = -1;
-
- public DataPartitionerWorker(JobConf job, String fnameNew, String fnameStaging, String[] fnamesPartitions, int start, int end)
- {
- _job = job;
- _fnameNew = fnameNew;
- _fnameStaging = fnameStaging;
- _fnamesPartitions = fnamesPartitions;
- _start = start;
- _end = end;
- }
-
- @Override
- public void run()
- {
- //read each input if required
- try
- {
- for( int i=_start; i<=_end; i++ )
- {
- String pdir = _fnamesPartitions[i];
- writeFileToHDFS( _job, _fnameNew, _fnameStaging+"/"+pdir );
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException("Failed on parallel data partitioning.", ex);
- }
- }
-
- public abstract void writeFileToHDFS( JobConf job, String fnameNew, String stagingDir )
- throws IOException;
- }
-
- private class DataPartitionerWorkerTextCell extends DataPartitionerWorker
- {
- public DataPartitionerWorkerTextCell(JobConf job, String fnameNew, String fnameStaging, String[] fnamesPartitions, int start, int end)
- {
- super(job, fnameNew, fnameStaging, fnamesPartitions, start, end);
- }
-
- @Override
- public void writeFileToHDFS(JobConf job, String fnameNew, String stagingDir)
- throws IOException
- {
- writeTextCellFileToHDFS( job, fnameNew, stagingDir );
- }
- }
-
- private class DataPartitionerWorkerBinaryCell extends DataPartitionerWorker
- {
- public DataPartitionerWorkerBinaryCell(JobConf job, String fnameNew, String fnameStaging, String[] fnamesPartitions, int start, int end)
- {
- super(job, fnameNew, fnameStaging, fnamesPartitions, start, end);
- }
-
- @Override
- public void writeFileToHDFS(JobConf job, String fnameNew, String stagingDir)
- throws IOException
- {
- writeBinaryCellSequenceFileToHDFS( job, fnameNew, stagingDir );
- }
- }
-
- private class DataPartitionerWorkerBinaryBlock extends DataPartitionerWorker
- {
- public DataPartitionerWorkerBinaryBlock(JobConf job, String fnameNew, String fnameStaging, String[] fnamesPartitions, int start, int end)
- {
- super(job, fnameNew, fnameStaging, fnamesPartitions, start, end);
- }
-
- @Override
- public void writeFileToHDFS(JobConf job, String fnameNew, String stagingDir)
- throws IOException
- {
- writeBinaryBlockSequenceFileToHDFS( job, fnameNew, stagingDir, true );
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
deleted file mode 100644
index d0cb7fc..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.lib.NullOutputFormat;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-/**
- * MR job class for submitting parfor remote partitioning MR jobs.
- *
- */
-public class DataPartitionerRemoteMR extends DataPartitioner
-{
-
- private long _pfid = -1;
- private int _numReducers = -1;
- private int _replication = -1;
- //private int _max_retry = -1;
- private boolean _jvmReuse = false;
- private boolean _keepIndexes = false;
-
-
- public DataPartitionerRemoteMR(PDataPartitionFormat dpf, int n, long pfid, int numReducers, int replication, int max_retry, boolean jvmReuse, boolean keepIndexes)
- {
- super(dpf, n);
-
- _pfid = pfid;
- _numReducers = numReducers;
- _replication = replication;
- //_max_retry = max_retry;
- _jvmReuse = jvmReuse;
- _keepIndexes = keepIndexes;
- }
-
-
- @Override
- protected void partitionMatrix(MatrixObject in, String fnameNew, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen)
- throws DMLRuntimeException
- {
- String jobname = "ParFor-DPMR";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- JobConf job;
- job = new JobConf( DataPartitionerRemoteMR.class );
- if( _pfid >= 0 ) //use in parfor
- job.setJobName(jobname+_pfid);
- else //use for partition instruction
- job.setJobName("Partition-MR");
-
- //maintain dml script counters
- Statistics.incrementNoOfCompiledMRJobs();
-
- try
- {
- //force writing to disk (typically not required since partitioning only applied if dataset exceeds CP size)
- in.exportData(); //written to disk iff dirty
-
- Path path = new Path(in.getFileName());
-
- /////
- //configure the MR job
- MRJobConfiguration.setPartitioningInfo(job, rlen, clen, brlen, bclen, ii, oi, _format, _n, fnameNew, _keepIndexes);
-
- //set mappers, reducers, combiners
- job.setMapperClass(DataPartitionerRemoteMapper.class);
- job.setReducerClass(DataPartitionerRemoteReducer.class);
-
- if( oi == OutputInfo.TextCellOutputInfo )
- {
- //binary cell intermediates for reduced IO
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(PairWritableCell.class);
- }
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- {
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(PairWritableCell.class);
- }
- else if ( oi == OutputInfo.BinaryBlockOutputInfo )
- {
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(PairWritableBlock.class);
-
- //check Alignment
- if( (_format == PDataPartitionFormat.ROW_BLOCK_WISE_N && rlen>_n && _n % brlen !=0)
- || (_format == PDataPartitionFormat.COLUMN_BLOCK_WISE_N && clen>_n && _n % bclen !=0) )
- {
- throw new DMLRuntimeException("Data partitioning format "+_format+" requires aligned blocks.");
- }
- }
-
- //set input format
- job.setInputFormat(ii.inputFormatClass);
-
- //set the input path and output path
- FileInputFormat.setInputPaths(job, path);
-
- //set output path
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
- //FileOutputFormat.setOutputPath(job, pathNew);
- job.setOutputFormat(NullOutputFormat.class);
-
- //////
- //set optimization parameters
-
- //set the number of mappers and reducers
- //job.setNumMapTasks( _numMappers ); //use default num mappers
- long reducerGroups = -1;
- switch( _format )
- {
- case ROW_WISE: reducerGroups = rlen; break;
- case COLUMN_WISE: reducerGroups = clen; break;
- case ROW_BLOCK_WISE: reducerGroups = (rlen/brlen)+((rlen%brlen==0)?0:1); break;
- case COLUMN_BLOCK_WISE: reducerGroups = (clen/bclen)+((clen%bclen==0)?0:1); break;
- case ROW_BLOCK_WISE_N: reducerGroups = (rlen/_n)+((rlen%_n==0)?0:1); break;
- case COLUMN_BLOCK_WISE_N: reducerGroups = (clen/_n)+((clen%_n==0)?0:1); break;
- default:
- //do nothing
- }
- job.setNumReduceTasks( (int)Math.min( _numReducers, reducerGroups) );
-
- //use FLEX scheduler configuration properties
- /*if( ParForProgramBlock.USE_FLEX_SCHEDULER_CONF )
- {
- job.setInt("flex.map.min", 0);
- job.setInt("flex.map.max", _numMappers);
- job.setInt("flex.reduce.min", 0);
- job.setInt("flex.reduce.max", _numMappers);
- }*/
-
- //disable automatic tasks timeouts and speculative task exec
- job.setInt("mapred.task.timeout", 0);
- job.setMapSpeculativeExecution(false);
-
- //set up preferred custom serialization framework for binary block format
- if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
- MRJobConfiguration.addBinaryBlockSerializationFramework( job );
-
- //enables the reuse of JVMs (multiple tasks per MR task)
- if( _jvmReuse )
- job.setNumTasksToExecutePerJvm(-1); //unlimited
-
- //enables compression - not conclusive for different codecs (empirically good compression ratio, but significantly slower)
- //job.set("mapred.compress.map.output", "true");
- //job.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
-
- //set the replication factor for the results
- job.setInt("dfs.replication", _replication);
-
- //set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
- DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
-
- //set the max number of retries per map task
- // disabled job-level configuration to respect cluster configuration
- // note: this refers to hadoop2, hence it never had effect on mr1
- //job.setInt("mapreduce.map.maxattempts", _max_retry);
-
- //set unique working dir
- MRJobConfiguration.setUniqueWorkingDir(job);
-
- /////
- // execute the MR job
- JobClient.runJob(job);
-
- //maintain dml script counters
- Statistics.incrementNoOfExecutedMRJobs();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- if( DMLScript.STATISTICS && _pfid >= 0 ){
- long t1 = System.nanoTime(); //only for parfor
- Statistics.maintainCPHeavyHitters("MR-Job_"+jobname, t1-t0);
- }
- }
-
-}
[29/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
deleted file mode 100644
index 799f8ab..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-
-/**
- * Rule: Split Hop DAG after CSV reads with unknown size. This is
- * important to create recompile hooks if format is read from mtd
- * (we are not able to split it on statementblock creation) and
- * mtd has unknown size (which can only happen for CSV).
- *
- */
-public class RewriteSplitDagUnknownCSVRead extends StatementBlockRewriteRule
-{
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- //collect all unknown csv reads hops
- ArrayList<Hop> cand = new ArrayList<Hop>();
- collectCSVReadHopsUnknownSize( sb.get_hops(), cand );
-
- //split hop dag on demand
- if( !cand.isEmpty() )
- {
- try
- {
- //duplicate sb incl live variable sets
- StatementBlock sb1 = new StatementBlock();
- sb1.setDMLProg(sb.getDMLProg());
- sb1.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- sb1.setLiveIn(new VariableSet());
- sb1.setLiveOut(new VariableSet());
-
- //move csv reads incl reblock to new statement block
- //(and replace original persistent read with transient read)
- ArrayList<Hop> sb1hops = new ArrayList<Hop>();
- for( Hop c : cand )
- {
- Hop reblock = c;
- long rlen = reblock.getDim1();
- long clen = reblock.getDim2();
- long nnz = reblock.getNnz();
- long brlen = reblock.getRowsInBlock();
- long bclen = reblock.getColsInBlock();
-
- //create new transient read
- DataOp tread = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(),
- DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, brlen, bclen);
- HopRewriteUtils.copyLineNumbers(reblock, tread);
-
- //replace reblock with transient read
- ArrayList<Hop> parents = new ArrayList<Hop>(reblock.getParent());
- for( int i=0; i<parents.size(); i++ )
- {
- Hop parent = parents.get(i);
- int pos = HopRewriteUtils.getChildReferencePos(parent, reblock);
- HopRewriteUtils.removeChildReferenceByPos(parent, reblock, pos);
- HopRewriteUtils.addChildReference(parent, tread, pos);
- }
-
- //add reblock sub dag to first statement block
- DataOp twrite = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(),
- reblock, DataOpTypes.TRANSIENTWRITE, null);
- twrite.setOutputParams(rlen, clen, nnz, brlen, bclen);
- HopRewriteUtils.copyLineNumbers(reblock, twrite);
- sb1hops.add(twrite);
-
- //update live in and out of new statement block (for piggybacking)
- DataIdentifier diVar = sb.variablesRead().getVariable(reblock.getName());
- if( diVar != null ){ //var read should always exist because persistent read
- sb1.liveOut().addVariable(reblock.getName(), new DataIdentifier(diVar));
- sb.liveIn().addVariable(reblock.getName(), new DataIdentifier(diVar));
- }
- }
-
- sb1.set_hops(sb1hops);
- sb1.updateRecompilationFlag();
- ret.add(sb1); //statement block with csv reblocks
- ret.add(sb); //statement block with remaining hops
- }
- catch(Exception ex)
- {
- throw new HopsException("Failed to split hops dag for csv read with unknown size.", ex);
- }
- LOG.debug("Applied splitDagUnknownCSVRead.");
- }
- //keep original hop dag
- else
- {
- ret.add(sb);
- }
-
- return ret;
- }
-
- /**
- *
- * @param roots
- * @param cand
- */
- private void collectCSVReadHopsUnknownSize( ArrayList<Hop> roots, ArrayList<Hop> cand )
- {
- if( roots == null )
- return;
-
- Hop.resetVisitStatus(roots);
- for( Hop root : roots )
- collectCSVReadHopsUnknownSize(root, cand);
- }
-
- /**
- *
- * @param root
- * @param cand
- */
- private void collectCSVReadHopsUnknownSize( Hop hop, ArrayList<Hop> cand )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //collect persistent reads (of type csv, with unknown size)
- if( hop instanceof DataOp )
- {
- DataOp dop = (DataOp) hop;
- if( dop.getDataOpType() == DataOpTypes.PERSISTENTREAD
- && dop.getInputFormatType() == FileFormatTypes.CSV
- && !dop.dimsKnown()
- && !HopRewriteUtils.hasOnlyWriteParents(dop, true, false)
- && !HopRewriteUtils.hasTransformParents(hop) )
- {
- cand.add(dop);
- }
- }
-
- //process children
- if( hop.getInput()!=null )
- for( Hop c : hop.getInput() )
- collectCSVReadHopsUnknownSize(c, cand);
-
- hop.setVisited(VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
deleted file mode 100644
index ee9c010..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-
-/**
- * Rule: Eliminate for Transient Write DataHops to have no parents
- * Solution: Move parent edges of Transient Write Hop to parent of
- * its child
- * Reason: Transient Write not being a root messes up
- * analysis for Lop's to Instruction translation (according to Amol)
- */
-public class RewriteTransientWriteParentHandling extends HopRewriteRule
-{
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- for (Hop h : roots)
- rule_RehangTransientWriteParents(h, roots);
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- // do nothing (does not apply to predicate hop DAGs)
- return root;
- }
-
-
- /**
- *
- * @param hop
- * @param sbHops
- * @throws HopsException
- */
- private void rule_RehangTransientWriteParents(Hop hop, ArrayList<Hop> sbHops)
- throws HopsException
- {
- if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE
- && !hop.getParent().isEmpty()) {
-
- // update parents inputs with data op input
- for (Hop p : hop.getParent()) {
- p.getInput().set(p.getInput().indexOf(hop), hop.getInput().get(0));
- }
-
- // update dataop input parent to add new parents except for
- // dataop itself
- hop.getInput().get(0).getParent().addAll(hop.getParent());
-
- // remove dataop parents
- hop.getParent().clear();
-
- // add dataop as root for this Hops DAG
- sbHops.add(hop);
-
- // do the same thing for my inputs (children)
- for (Hop hi : hop.getInput()) {
- rule_RehangTransientWriteParents(hi,sbHops);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
deleted file mode 100644
index 0e4c7b8..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.StatementBlock;
-
-/**
- * Base class for all hop rewrites in order to enable generic
- * application of all rules.
- *
- */
-public abstract class StatementBlockRewriteRule
-{
-
- protected static final Log LOG = LogFactory.getLog(StatementBlockRewriteRule.class.getName());
-
- /**
- * Handle an arbitrary statement block. Specific type constraints have to be ensured
- * within the individual rewrites.
- *
- * @param sb
- * @param sate
- * @return
- * @throws HopsException
- */
- public abstract ArrayList<StatementBlock> rewriteStatementBlock( StatementBlock sb, ProgramRewriteStatus sate )
- throws HopsException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Aggregate.java b/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
deleted file mode 100644
index 88d57e7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to represent an aggregation.
- * It is used in rowsum, colsum, etc.
- */
-
-public class Aggregate extends Lop
-{
-
-
- /** Aggregate operation types **/
-
- public enum OperationTypes {
- Sum, Product, Min, Max, Trace, KahanSum, KahanSumSq, KahanTrace, Mean,MaxIndex, MinIndex
- }
- OperationTypes operation;
-
- private boolean isCorrectionUsed = false;
- private CorrectionLocationType correctionLocation = CorrectionLocationType.INVALID;
-
- /**
- * @param input - input lop
- * @param op - operation type
- */
- public Aggregate(Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt ) {
- super(Lop.Type.Aggregate, dt, vt);
- init ( input, op, dt, vt, ExecType.MR );
- }
-
- public Aggregate(Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt, ExecType et ) {
- super(Lop.Type.Aggregate, dt, vt);
- init ( input, op, dt, vt, et );
- }
-
- private void init (Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt, ExecType et ) {
- operation = op;
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- // this function must be invoked during hop-to-lop translation
- public void setupCorrectionLocation(CorrectionLocationType loc) {
- if (operation == OperationTypes.KahanSum || operation == OperationTypes.KahanSumSq
- || operation == OperationTypes.KahanTrace || operation == OperationTypes.Mean) {
- isCorrectionUsed = true;
- correctionLocation = loc;
- }
- }
-
- /**
- * for debugging purposes.
- */
-
- public String toString()
- {
- return "Operation: " + operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
-
- private String getOpcode() {
- switch(operation) {
- case Sum:
- case Trace:
- return "a+";
- case Mean:
- return "amean";
- case Product:
- return "a*";
- case Min:
- return "amin";
- case Max:
- return "amax";
- case MaxIndex:
- return "arimax";
- case MinIndex:
- return "arimin";
- case KahanSum:
- case KahanTrace:
- return "ak+";
- case KahanSumSq:
- return "asqk+";
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Aggregate operation: " + operation);
- }
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- String opcode = getOpcode();
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( opcode );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- boolean isCorrectionApplicable = false;
-
- String opcode = getOpcode();
- if (operation == OperationTypes.Mean || operation == OperationTypes.KahanSum
- || operation == OperationTypes.KahanSumSq || operation == OperationTypes.KahanTrace)
- isCorrectionApplicable = true;
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( opcode );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- if ( isCorrectionApplicable )
- {
- // add correction information to the instruction
- sb.append( OPERAND_DELIMITOR );
- sb.append( isCorrectionUsed );
- sb.append( OPERAND_DELIMITOR );
- sb.append( correctionLocation );
- }
-
- return sb.toString();
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendCP.java b/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
deleted file mode 100644
index f3108e5..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendCP extends Lop
-{
- public static final String OPCODE = "append";
-
- private boolean _cbind = true;
-
- public AppendCP(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
-
- return " AppendCP: ";
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( "append" );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendG.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendG.java b/src/main/java/com/ibm/bi/dml/lops/AppendG.java
deleted file mode 100644
index f254aa0..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendG.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-/**
- * TODO Additional compiler enhancements:
- * 1) Partial Shuffle Elimination - Any full or aligned blocks could be directly output from the mappers
- * to the result index. We only need to shuffle, sort and aggregate partial blocks. However, this requires
- * piggybacking changes, i.e., (1) operations with multiple result indexes, and (2) multiple operations
- * with the same result index.
- * 2) Group Elimination for Append Chains - If we have chains of rappend each intermediate is shuffled and
- * aggregated. This is unnecessary if all offsets are known in advance. We could directly pack all rappends
- * in one GMR map-phase followed by one group and subsequent aggregate. However, this requires an n-ary
- * rappend or group (with multiple inputs).
- *
- */
-public class AppendG extends Lop
-{
- public static final String OPCODE = "gappend";
-
- private boolean _cbind = true;
-
- public AppendG(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, boolean cbind, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, input4, dt, vt, et);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- addInput(input4);
- input4.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SP
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return " AppendG: ";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(input_index3),
- String.valueOf(input_index4),
- String.valueOf(output_index) );
- }
-
- //called when append executes in SP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String input_index4, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(3).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java b/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
deleted file mode 100644
index b1b86c1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-public class AppendGAlignedSP extends Lop
-{
- public static final String OPCODE = "galignedappend";
-
- private boolean _cbind = true;
-
- public AppendGAlignedSP(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt);
-
- _cbind = true;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return " AppendGSP: ";
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendM.java b/src/main/java/com/ibm/bi/dml/lops/AppendM.java
deleted file mode 100644
index ab364ca..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendM.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendM extends Lop
-{
- public static final String OPCODE = "mappend";
-
- public enum CacheType {
- RIGHT,
- RIGHT_PART,
- }
-
- private boolean _cbind = true;
- private CacheType _cacheType = null;
-
- public AppendM(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind, boolean partitioned, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt, et);
-
- _cbind = cbind;
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return "Operation = AppendM";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(input_index3),
- String.valueOf(output_index) );
- }
-
-
- //called when append executes in SP
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output) );
-
- //note: for SP: no cache type
- if( getExecType()==ExecType.MR ){
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheType);
- }
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-
- public boolean usesDistributedCache() {
- return true;
- }
-
- public int[] distributedCacheInputIndex() {
- return new int[]{2}; // second input is from distributed cache
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendR.java b/src/main/java/com/ibm/bi/dml/lops/AppendR.java
deleted file mode 100644
index 94b83bf..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendR.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendR extends Lop
-{
- public static final String OPCODE = "rappend";
-
- private boolean _cbind = true;
-
- public AppendR(Lop input1, Lop input2, DataType dt, ValueType vt, boolean cbind, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, dt, vt, et);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN); //currently required for correctness
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //SP
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return " AppendR: ";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(output_index) );
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Binary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Binary.java b/src/main/java/com/ibm/bi/dml/lops/Binary.java
deleted file mode 100644
index 1ceabda..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Binary.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform binary operation. Both inputs must be matrices or vectors.
- * Example - A = B + C, where B and C are matrices or vectors.
- */
-
-public class Binary extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, MULTIPLY, DIVIDE, MINUS1_MULTIPLY, MODULUS, INTDIV, MATMULT,
- LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS,
- AND, OR,
- MAX, MIN, POW, SOLVE, NOTSUPPORTED
- };
-
- private OperationTypes operation;
- private int numThreads = -1;
-
- /**
- * Constructor to perform a binary operation.
- * @param input
- * @param op
- */
- public Binary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- this(input1, input2, op, dt, vt, et, 1);
- }
-
- public Binary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et, int k) {
- super(Lop.Type.Binary, dt, vt);
- init(input1, input2, op, dt, vt, et);
- numThreads = k;
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else if ( et == ExecType.CP || et == ExecType.SPARK ){
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
-
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- private String getOpcode()
- {
- return getOpcode( operation );
- }
-
- public static String getOpcode( OperationTypes op ) {
- switch(op) {
- /* Arithmetic */
- case ADD:
- return "+";
- case SUBTRACT:
- return "-";
- case MULTIPLY:
- return "*";
- case DIVIDE:
- return "/";
- case MODULUS:
- return "%%";
- case INTDIV:
- return "%/%";
- case MATMULT:
- return "ba+*";
- case MINUS1_MULTIPLY:
- return "1-*";
-
- /* Relational */
- case LESS_THAN:
- return "<";
- case LESS_THAN_OR_EQUALS:
- return "<=";
- case GREATER_THAN:
- return ">";
- case GREATER_THAN_OR_EQUALS:
- return ">=";
- case EQUALS:
- return "==";
- case NOT_EQUALS:
- return "!=";
-
- /* Boolean */
- case AND:
- return "&&";
- case OR:
- return "||";
-
-
- /* Builtin Functions */
- case MIN:
- return "min";
- case MAX:
- return "max";
- case POW:
- return "^";
-
- case SOLVE:
- return "solve";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for Binary operation: " + op);
- }
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- //append degree of parallelism for matrix multiplications
- if( operation == OperationTypes.MATMULT && getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException
- {
- return getInstructions(input_index1+"", input_index2+"", output_index+"");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryM.java b/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
deleted file mode 100644
index 5b03008..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.Binary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform binary operation. Both inputs must be matrices or vectors.
- * Example - A = B + C, where B and C are matrices or vectors.
- */
-
-public class BinaryM extends Lop
-{
-
- public enum CacheType {
- RIGHT,
- RIGHT_PART,
- }
-
- public enum VectorType{
- COL_VECTOR,
- ROW_VECTOR,
- }
-
- private OperationTypes _operation;
- private CacheType _cacheType = null;
- private VectorType _vectorType = null;
-
- /**
- * Constructor to perform a binary operation.
- * @param input
- * @param op
- * @throws DMLRuntimeException
- */
-
- public BinaryM(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean partitioned, boolean colVector ) throws LopsException {
- super(Lop.Type.Binary, dt, vt);
-
- _operation = op;
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- _vectorType = colVector ? VectorType.COL_VECTOR : VectorType.ROW_VECTOR;
-
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if(et == ExecType.MR) {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else if(et == ExecType.SPARK) {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- else {
- throw new LopsException("Incorrect execution type for BinaryM lop:" + et.name());
- }
- }
-
-
- @Override
- public String toString()
- {
- return " Operation: " + _operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return _operation;
- }
-
- private String getOpcode()
- {
- return getOpcode( _operation );
- }
-
- public static String getOpcode( OperationTypes op ) {
- switch(op) {
- /* Arithmetic */
- case ADD:
- return "map+";
- case SUBTRACT:
- return "map-";
- case MULTIPLY:
- return "map*";
- case DIVIDE:
- return "map/";
- case MODULUS:
- return "map%%";
- case INTDIV:
- return "map%/%";
- case MINUS1_MULTIPLY:
- return "map1-*";
-
- /* Relational */
- case LESS_THAN:
- return "map<";
- case LESS_THAN_OR_EQUALS:
- return "map<=";
- case GREATER_THAN:
- return "map>";
- case GREATER_THAN_OR_EQUALS:
- return "map>=";
- case EQUALS:
- return "map==";
- case NOT_EQUALS:
- return "map!=";
-
- /* Boolean */
- case AND:
- return "map&&";
- case OR:
- return "map||";
-
-
- /* Builtin Functions */
- case MIN:
- return "mapmin";
- case MAX:
- return "mapmax";
- case POW:
- return "map^";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for Binary operation: " + op);
- }
- }
-
- public static boolean isOpcode(String opcode)
- {
- return opcode.equals("map+") || opcode.equals("map-") ||
- opcode.equals("map*") || opcode.equals("map/") ||
- opcode.equals("map%%") || opcode.equals("map%/%") ||
- opcode.equals("map<") || opcode.equals("map<=") ||
- opcode.equals("map>") || opcode.equals("map>=") ||
- opcode.equals("map==") || opcode.equals("map!=") ||
- opcode.equals("map&&") || opcode.equals("map||") ||
- opcode.equals("mapmin") || opcode.equals("mapmax") ||
- opcode.equals("map^") || opcode.equals("map1-*");
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(0).prepInputOperand(input1));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(1).prepInputOperand(input2));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_cacheType);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_vectorType);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException
- {
- return getInstructions(input_index1+"", input_index2+"", output_index+"");
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- // second input is from distributed cache
- return new int[]{2};
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java b/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
deleted file mode 100644
index 6fbe4a5..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-/**
- * Lop to perform binary scalar operations. Both inputs must be scalars.
- * Example i = j + k, i = i + 1.
- */
-
-public class BinaryScalar extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, SUBTRACTRIGHT, MULTIPLY, DIVIDE, MODULUS, INTDIV,
- LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS,
- AND, OR,
- LOG,POW,MAX,MIN,PRINT,
- IQSIZE,
- Over,
- SEQINCR
- }
-
- OperationTypes operation;
-
- /**
- * This overloaded constructor is used for setting exec type in case of spark backend
- */
- public BinaryScalar(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- super(Lop.Type.BinaryCP, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false; // this field does not carry any meaning for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- /**
- * Constructor to perform a scalar operation
- * @param input
- * @param op
- */
-
- public BinaryScalar(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.BinaryCP, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false; // this field does not carry any meaning for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return "Operation: " + operation;
- }
-
- public OperationTypes getOperationType(){
- return operation;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output) throws LopsException
- {
- String opString = getOpcode( operation );
-
-
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( opString );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public Lop.SimpleInstType getSimpleInstructionType()
- {
- switch (operation){
-
- default:
- return SimpleInstType.Scalar;
- }
- }
-
- /**
- *
- * @param op
- * @return
- */
- public static String getOpcode( OperationTypes op )
- {
- switch ( op )
- {
- /* Arithmetic */
- case ADD:
- return "+";
- case SUBTRACT:
- return "-";
- case MULTIPLY:
- return "*";
- case DIVIDE:
- return "/";
- case MODULUS:
- return "%%";
- case INTDIV:
- return "%/%";
- case POW:
- return "^";
-
- /* Relational */
- case LESS_THAN:
- return "<";
- case LESS_THAN_OR_EQUALS:
- return "<=";
- case GREATER_THAN:
- return ">";
- case GREATER_THAN_OR_EQUALS:
- return ">=";
- case EQUALS:
- return "==";
- case NOT_EQUALS:
- return "!=";
-
- /* Boolean */
- case AND:
- return "&&";
- case OR:
- return "||";
-
- /* Builtin Functions */
- case LOG:
- return "log";
- case MIN:
- return "min";
- case MAX:
- return "max";
-
- case PRINT:
- return "print";
-
- case IQSIZE:
- return "iqsize";
-
- case SEQINCR:
- return "seqincr";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for BinaryScalar operator: " + op);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java b/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
deleted file mode 100644
index d6b9171..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class BinaryUAggChain extends Lop
-{
-
- public static final String OPCODE = "binuaggchain";
-
- //outer operation
- private Binary.OperationTypes _binOp = null;
- //inner operation
- private Aggregate.OperationTypes _uaggOp = null;
- private PartialAggregate.DirectionTypes _uaggDir = null;
-
-
- /**
- * Constructor to setup a map mult chain without weights
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public BinaryUAggChain(Lop input1, Binary.OperationTypes bop, Aggregate.OperationTypes uaop, PartialAggregate.DirectionTypes uadir, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.BinUaggChain, dt, vt);
- addInput(input1); //X
- input1.addOutput(this);
-
- //setup operator types
- _binOp = bop;
- _uaggOp = uaop;
- _uaggDir = uadir;
-
- //setup MR parameters
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
-
- public String toString() {
- return "Operation = BinUaggChain";
- }
-
- @Override
- public String getInstructions(int input_index1, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java b/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
deleted file mode 100644
index e856de6..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-
-
-/**
- * Lop to convert CSV data into SystemML data format (TextCell, BinaryCell, or BinaryBlock)
- */
-public class CSVReBlock extends Lop
-{
-
- public static final String OPCODE = "csvrblk";
-
- Long rows_per_block;
- Long cols_per_block;
-
- public CSVReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, ExecType et) throws LopsException
- {
- super(Lop.Type.CSVReBlock, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- this.rows_per_block = rows_per_block;
- this.cols_per_block = cols_per_block;
-
- /*
- * This lop can be executed only in CSVREBLOCK job.
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- // If the input to reblock is a tranform, then piggyback it along with transform
- if ( input instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)input).getOp() == OperationTypes.TRANSFORM )
- {
- definesMRJob = false;
- lps.addCompatibility(JobType.TRANSFORM);
- }
- else
- {
- lps.addCompatibility(JobType.CSV_REBLOCK);
- }
-
- if(et == ExecType.MR) {
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- }
- else if(et == ExecType.SPARK) {
- this.lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- else {
- throw new LopsException("Incorrect execution type for CSVReblock:" + et);
- }
- }
-
- @Override
- public String toString() {
-
- return "CSVReblock - rows per block = " + rows_per_block + " cols per block " + cols_per_block ;
- }
-
- private String prepCSVProperties() throws LopsException {
- StringBuilder sb = new StringBuilder();
-
- boolean isSparkTransformInput = false;
- Data dataInput = null;
- if(getInputs().get(0).getType() == Type.Data)
- dataInput = (Data)getInputs().get(0);
- else if ( getInputs().get(0).getType() == Type.ParameterizedBuiltin && ((ParameterizedBuiltin)getInputs().get(0)).getOp() == OperationTypes.TRANSFORM) {
- isSparkTransformInput = (getExecType() == ExecType.SPARK);
- dataInput = (Data) ((ParameterizedBuiltin)getInputs().get(0)).getNamedInput(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_DATA);
- }
-
- Lop headerLop = dataInput.getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Lop delimLop = dataInput.getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Lop fillLop = dataInput.getNamedInputLop(DataExpression.DELIM_FILL);
- Lop fillValueLop = dataInput.getNamedInputLop(DataExpression.DELIM_FILL_VALUE);
-
- if (headerLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_HAS_HEADER_ROW
- + " must be a literal.");
- if (delimLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_DELIMITER
- + " must be a literal.");
- if (fillLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_FILL
- + " must be a literal.");
- if (fillValueLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_FILL_VALUE
- + " must be a literal.");
-
- // Output from transform() does not have a header
- // On MR, reblock is piggybacked along with transform, and hence
- // specific information about header needn't be passed through instruction
- sb.append( ((Data)headerLop).getBooleanValue() && !isSparkTransformInput );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)delimLop).getStringValue() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)fillLop).getBooleanValue() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)fillValueLop).getDoubleValue() );
-
- return sb.toString();
-}
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
-
- Lop input = getInputs().get(0);
-
- sb.append( input.prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( cols_per_block );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepCSVProperties() );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("The method getInstructions(String,String) for CSVReblock should be called only for Spark execution type");
- }
-
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( cols_per_block );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepCSVProperties() );
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for CSVReblock operation");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java b/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
deleted file mode 100644
index b7ab685..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class CentralMoment extends Lop
-{
-
- /**
- * Constructor to perform central moment.
- * input1 <- data (weighted or unweighted)
- * input2 <- order (integer: 0, 2, 3, or 4)
- */
-
- private void init(Lop input1, Lop input2, Lop input3, ExecType et) {
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- if ( et == ExecType.MR ) {
- definesMRJob = true;
- lps.addCompatibility(JobType.CM_COV);
- lps.setProperties(inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- }
- else { //CP/SPARK
- // when executing in CP, this lop takes an optional 3rd input (Weights)
- if ( input3 != null ) {
- this.addInput(input3);
- input3.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public CentralMoment(Lop input1, Lop input2, DataType dt, ValueType vt) {
- this(input1, input2, null, dt, vt, ExecType.MR);
- }
-
- public CentralMoment(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) {
- this(input1, input2, null, dt, vt, et);
- }
-
- public CentralMoment(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
- this(input1, input2, input3, dt, vt, ExecType.MR);
- }
-
- public CentralMoment(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.CentralMoment, dt, vt);
- init(input1, input2, input3, et);
- }
-
- @Override
- public String toString() {
-
- return "Operation = CentralMoment";
- }
-
- /**
- * Function to generate CP centralMoment instruction for weighted operation.
- *
- * input1: data
- * input2: weights
- * input3: order
- */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( "cm" );
- sb.append( OPERAND_DELIMITOR );
-
- // Input data
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
-
- // Weights
- sb.append( getInputs().get(1).prepInputOperand(input2) );
- sb.append( OPERAND_DELIMITOR );
-
- // Order
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate CP centralMoment instruction for unweighted operation.
- *
- * input1: data
- * input2: order (not used, and order is derived internally!)
- */
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( "cm" );
- sb.append( OPERAND_DELIMITOR );
-
- // Input data (can be weighted or unweighted)
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- // Order
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate MR central moment instruction, invoked from
- * <code>Dag.java:getAggAndOtherInstructions()</code>. This function
- * is used for both weighted and unweighted operations.
- *
- * input_index: data (in case of weighted: data combined via combinebinary)
- * output_index: produced output
- *
- * The order for central moment is derived internally in the function.
- */
- @Override
- public String getInstructions(int input_index, int output_index) {
- return getInstructions(input_index+"", "", output_index+"");
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java b/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
deleted file mode 100644
index e74699b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import org.apache.spark.storage.StorageLevel;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop for checkpoint operations. For example, on Spark, the semantic of a checkpoint
- * is to persist an intermediate result into a specific storage level (e.g., mem_only).
- *
- * We use the name checkpoint in terms of cache/persist in Spark (not in terms of checkpoint
- * in Spark streaming) in order to differentiate from CP caching.
- *
- * NOTE: since this class uses spark apis, it should only be instantiated if we are
- * running in execution mode spark (whenever all spark libraries are available)
- */
-public class Checkpoint extends Lop
-{
- public static final String OPCODE = "chkpoint";
-
- public static final StorageLevel DEFAULT_STORAGE_LEVEL = StorageLevel.MEMORY_AND_DISK();
- public static final StorageLevel SER_STORAGE_LEVEL = StorageLevel.MEMORY_AND_DISK_SER();
- public static final String STORAGE_LEVEL = "storage.level";
-
- private StorageLevel _storageLevel;
-
-
- /**
- * TODO change string parameter storage.level to StorageLevel as soon as we can assume
- * that Spark libraries are always available.
- *
- * @param input
- * @param dt
- * @param vt
- * @param level
- * @param et
- * @throws LopsException
- */
- public Checkpoint(Lop input, DataType dt, ValueType vt, String level)
- throws LopsException
- {
- super(Lop.Type.Checkpoint, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- _storageLevel = StorageLevel.fromString(level);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- public StorageLevel getStorageLevel()
- {
- return _storageLevel;
- }
-
- public void setStorageLevel(StorageLevel level)
- {
- _storageLevel = level;
- }
-
- @Override
- public String toString() {
- return "Checkpoint - storage.level = " + _storageLevel.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- //valid execution type
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("Wrong execution type for Checkpoint.getInstructions (expected: SPARK, found: "+getExecType()+").");
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getStorageLevelString(_storageLevel) );
-
- return sb.toString();
-
- }
-
- /**
- * This is a utility method because Sparks StorageLevel.toString() is incompatible with its own
- * fromString() method.
- *
- * @param level
- * @return
- */
- public static String getStorageLevelString( StorageLevel level)
- {
- if( StorageLevel.NONE().equals(level) )
- return "NONE";
- else if( StorageLevel.MEMORY_ONLY().equals(level) )
- return "MEMORY_ONLY";
- else if( StorageLevel.MEMORY_ONLY_2().equals(level) )
- return "MEMORY_ONLY_2";
- else if( StorageLevel.MEMORY_ONLY_SER().equals(level) )
- return "MEMORY_ONLY_SER";
- else if( StorageLevel.MEMORY_ONLY_SER_2().equals(level) )
- return "MEMORY_ONLY_SER_2";
- else if( StorageLevel.MEMORY_AND_DISK().equals(level) )
- return "MEMORY_AND_DISK";
- else if( StorageLevel.MEMORY_AND_DISK_2().equals(level) )
- return "MEMORY_AND_DISK_2";
- else if( StorageLevel.MEMORY_AND_DISK_SER().equals(level) )
- return "MEMORY_AND_DISK_SER";
- else if( StorageLevel.MEMORY_AND_DISK_SER_2().equals(level) )
- return "MEMORY_AND_DISK_SER_2";
- else if( StorageLevel.DISK_ONLY().equals(level) )
- return "DISK_ONLY";
- else if( StorageLevel.DISK_ONLY_2().equals(level) )
- return "DISK_ONLY_2";
-
- return "INVALID";
- }
-
- /**
- *
- * @return
- */
- public static String getDefaultStorageLevelString() {
- return getStorageLevelString( DEFAULT_STORAGE_LEVEL );
- }
-
- /**
- *
- * @return
- */
- public static String getSerializeStorageLevelString() {
- return getStorageLevelString( SER_STORAGE_LEVEL );
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CoVariance.java b/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
deleted file mode 100644
index 2d1626e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Lop to compute covariance between two 1D matrices
- *
- */
-public class CoVariance extends Lop
-{
-
-
- /**
- * Constructor to perform covariance.
- * input1 <- data
- * (prior to this lop, input vectors need to attached together using CombineBinary or CombineTertiary)
- * @throws LopsException
- */
-
- public CoVariance(Lop input1, DataType dt, ValueType vt) throws LopsException {
- this(input1, dt, vt, ExecType.MR);
- }
-
- public CoVariance(Lop input1, DataType dt, ValueType vt, ExecType et) throws LopsException {
- super(Lop.Type.CoVariance, dt, vt);
- init(input1, null, null, et);
- }
-
- public CoVariance(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) throws LopsException {
- this(input1, input2, null, dt, vt, et);
- }
-
- public CoVariance(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et) throws LopsException {
- super(Lop.Type.CoVariance, dt, vt);
- init(input1, input2, input3, et);
- }
-
- private void init(Lop input1, Lop input2, Lop input3, ExecType et)
- throws LopsException
- {
- /*
- * When et = MR: covariance lop will have a single input lop, which
- * denote the combined input data -- output of combinebinary, if unweighed;
- * and output combineteriaty (if weighted).
- *
- * When et = CP: covariance lop must have at least two input lops, which
- * denote the two input columns on which covariance is computed. It also
- * takes an optional third arguments, when weighted covariance is computed.
- */
- addInput(input1);
- input1.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- if ( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.CM_COV);
- lps.setProperties(inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- }
- else //CP/SPARK
- {
- definesMRJob = false;
- if ( input2 == null ) {
- throw new LopsException(this.printErrorLocation() + "Invalid inputs to covariance lop.");
- }
- addInput(input2);
- input2.addOutput(this);
-
- if ( input3 != null ) {
- addInput(input3);
- input3.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- @Override
- public String toString() {
-
- return "Operation = coVariance";
- }
-
- /**
- * Function two generate CP instruction to compute unweighted covariance.
- * input1 -> input column 1
- * input2 -> input column 2
- */
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function two generate CP instruction to compute weighted covariance.
- * input1 -> input column 1
- * input2 -> input column 2
- * input3 -> weights
- */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input3));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate MR version of covariance instruction.
- * input_index -> denote the "combined" input columns and weights,
- * when applicable.
- */
- @Override
- public String getInstructions(int input_index, int output_index) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( this.prepInputOperand(output_index));
-
- return sb.toString();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java b/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
deleted file mode 100644
index 81fafef..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to represent an combine operation -- used ONLY in the context of sort.
- */
-
-public class CombineBinary extends Lop
-{
-
-
- public enum OperationTypes {PreSort, PreCentralMoment, PreCovUnweighted, PreGroupedAggUnweighted}; // (PreCovWeighted,PreGroupedAggWeighted) will be CombineTertiary
- OperationTypes operation;
-
- /**
- * @param input - input lop
- * @param op - operation type
- */
-
- public CombineBinary(OperationTypes op, Lop input1, Lop input2, DataType dt, ValueType vt)
- {
- super(Lop.Type.CombineBinary, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- /*
- * This lop can ONLY be executed as a STANDALONE job
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- lps.addCompatibility(JobType.COMBINE);
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- this.lps.setProducesIntermediateOutput(true);
- }
-
- public String toString()
- {
- return "combinebinary";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- // Determine whether or not the second input denotes weights vector.
- // CombineBinary can be used to combine (data,weights) vectors or (data1,data2) vectors
- boolean isSecondInputIsWeight = true;
- if ( operation == OperationTypes.PreCovUnweighted || operation == OperationTypes.PreGroupedAggUnweighted ) {
- isSecondInputIsWeight = false;
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "combinebinary" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOperand(String.valueOf(isSecondInputIsWeight), DataType.SCALAR, ValueType.BOOLEAN) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- public OperationTypes getOperation() {
- return operation;
- }
-
- public static CombineBinary constructCombineLop(OperationTypes op, Lop input1,
- Lop input2, DataType dt, ValueType vt) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
-
- // find intersection of input1.getOutputs() and input2.getOutputs();
- set1.retainAll(input2.getOutputs());
-
- for (Lop lop : set1) {
- if ( lop.type == Lop.Type.CombineBinary ) {
- CombineBinary combine = (CombineBinary)lop;
- if ( combine.operation == op)
- return (CombineBinary)lop;
- }
- }
-
- CombineBinary comn = new CombineBinary(op, input1, input2, dt, vt);
- comn.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return comn;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java b/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
deleted file mode 100644
index 529d57b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to represent an combine operation -- used ONLY in the context of sort.
- */
-
-public class CombineTernary extends Lop
-{
-
- public enum OperationTypes {
- PreCovWeighted, PreGroupedAggWeighted
- }; // PreCovUnweighted,PreGroupedAggWeighted will be CombineBinary
-
- OperationTypes operation;
-
- public CombineTernary( OperationTypes op, Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
- super(Lop.Type.CombineTernary, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- this.addInput(input3);
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- /*
- * This lop can ONLY be executed as a STANDALONE job
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- this.lps.addCompatibility(JobType.COMBINE);
- this.lps.setProperties(inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- this.lps.setProducesIntermediateOutput(true);
- }
-
- public String toString() {
- return "combineternary";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2,
- int input_index3, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "combineternary" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepInputOperand(output_index));
-
- return sb.toString();
- }
-
- public OperationTypes getOperation() {
- return operation;
- }
-
- public static CombineTernary constructCombineLop( OperationTypes op, Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
-
- // find intersection of input1.getOutputs() and input2.getOutputs()
- set1.retainAll(input2.getOutputs());
-
- // find intersection of the above result and input3.getOutputs()
- set1.retainAll(input3.getOutputs());
-
- for (Lop lop : set1) {
- if (lop.type == Lop.Type.CombineTernary) {
- CombineTernary combine = (CombineTernary) lop;
- if (combine.operation == op)
- return (CombineTernary) lop;
- }
- }
-
- CombineTernary comn = new CombineTernary(op, input1, input2, input3, dt, vt);
- comn.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return comn;
- }
-
-}
[70/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
index 7d1442d..0ac8655 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
@@ -15,64 +15,64 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.Optimizer.PlanInputType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.DataGenCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cpfile.MatrixIndexingCPFileInstruction;
-import com.ibm.bi.dml.runtime.instructions.cpfile.ParameterizedBuiltinCPFileInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.PlanInputType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
+import org.apache.sysml.runtime.instructions.cpfile.MatrixIndexingCPFileInstruction;
+import org.apache.sysml.runtime.instructions.cpfile.ParameterizedBuiltinCPFileInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* Converter for creating an internal plan representation for a given runtime program
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java
index 43f2227..fdfd8b7 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java
@@ -15,35 +15,35 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java
index 1cfcfbd..1f845bb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.Map;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
/**
* Helper class for mapping nodes of the internal plan representation to statement blocks and
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
index c2f9925..2ee20b9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.Map;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public class OptTreePlanMappingAbstract extends OptTreePlanMapping
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
index 8ed21c3..8d6c0f2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.instructions.Instruction;
public class OptTreePlanMappingRuntime extends OptTreePlanMapping
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
index 059e54c..bcf0fed 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
import java.util.HashMap;
@@ -28,46 +28,46 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ipa.InterProceduralAnalysis;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteRule;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriteStatus;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteConstantFolding;
-import com.ibm.bi.dml.hops.rewrite.RewriteRemoveUnnecessaryBranches;
-import com.ibm.bi.dml.hops.rewrite.StatementBlockRewriteRule;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.Optimizer.CostModelType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ipa.InterProceduralAnalysis;
+import org.apache.sysml.hops.rewrite.HopRewriteRule;
+import org.apache.sysml.hops.rewrite.ProgramRewriteStatus;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.hops.rewrite.RewriteConstantFolding;
+import org.apache.sysml.hops.rewrite.RewriteRemoveUnnecessaryBranches;
+import org.apache.sysml.hops.rewrite.StatementBlockRewriteRule;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.CostModelType;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.Statistics;
/**
@@ -97,7 +97,7 @@ public class OptimizationWrapper
{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.runtime.controlprogram.parfor.opt")
+ Logger.getLogger("org.apache.sysml.runtime.controlprogram.parfor.opt")
.setLevel((Level) Level.DEBUG);
}
}
@@ -186,7 +186,7 @@ public class OptimizationWrapper
public static void setLogLevel( Level optLogLevel )
{
if( !LDEBUG ){ //set log level if not overwritten by internal flag
- Logger.getLogger("com.ibm.bi.dml.runtime.controlprogram.parfor.opt")
+ Logger.getLogger("org.apache.sysml.runtime.controlprogram.parfor.opt")
.setLevel( optLogLevel );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
index e1a16c6..e19932a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.Collection;
import java.util.LinkedList;
@@ -23,15 +23,15 @@ import java.util.LinkedList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index 09431cc..8081159 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.HashSet;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
/**
* Rule-Based ParFor Optimizer (time: O(n)):
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
index 8294b26..a3680a2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index ae581e6..89cf653 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,69 +28,69 @@ import java.util.Set;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.AggBinaryOp.MMultMethod;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriteStatus;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteInjectSparkLoopCheckpointing;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalFile;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRow;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.AggBinaryOp.MMultMethod;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.hops.rewrite.ProgramRewriteStatus;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.hops.rewrite.RewriteInjectSparkLoopCheckpointing;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.ResultMergeLocalFile;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/**
* Rule-Based ParFor Optimizer (time: O(n)):
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
index 4f46c78..a050d7d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.InternalTestVariable;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.InternalTestVariable;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
/**
* Internal representation of a test configuration consisting of a logical TestMeasure,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
index b08a051..fffc281 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Matrix.ValueType;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.udf.FunctionParameter;
+import org.apache.sysml.udf.Matrix;
+import org.apache.sysml.udf.PackageFunction;
+import org.apache.sysml.udf.PackageRuntimeException;
+import org.apache.sysml.udf.Matrix.ValueType;
/**
* External function (type CP) used within the PerfTestTool in order to
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
index b3dce1e..409cbf9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.lang.ref.WeakReference;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
index 602162a..987af19 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.io.BufferedReader;
import java.io.File;
@@ -45,45 +45,45 @@ import org.apache.commons.logging.LogFactory;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.LazyWriteBuffer;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DataGenCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
* DML Instructions Performance Test Tool:
@@ -123,7 +123,7 @@ public class PerfTestTool
private static final String PERF_TOOL_DIR = "./conf/PerfTestTool/";
// private static final String PERF_RESULTS_FNAME = PERF_TOOL_DIR + "%id%.dat";
private static final String PERF_PROFILE_FNAME = PERF_TOOL_DIR + "performance_profile.xml";
- private static final String DML_SCRIPT_FNAME = "./src/com/ibm/bi/dml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml";
+ private static final String DML_SCRIPT_FNAME = "./src/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml";
private static final String DML_TMP_FNAME = PERF_TOOL_DIR + "temp.dml";
//XML profile tags and attributes
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml
index f0d563a..a401a78 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestToolRegression.dml
@@ -21,15 +21,15 @@
dynRead = externalFunction(Matrix[Double] d, String fname, Integer m, Integer n)
return (Matrix[Double] D)
-implemented in (classname="com.ibm.bi.dml.runtime.controlprogram.parfor.test.dml.DynamicReadMatrix2DCP",exectype="mem")
+implemented in (classname="org.apache.sysml.runtime.controlprogram.parfor.test.dml.DynamicReadMatrix2DCP",exectype="mem")
dynWrite = externalFunction(Matrix[Double] R, String fname)
return (Matrix[Double] D)
-implemented in (classname="com.ibm.bi.dml.runtime.controlprogram.parfor.test.dml.DynamicWriteMatrix2DCP",exectype="mem")
+implemented in (classname="org.apache.sysml.runtime.controlprogram.parfor.test.dml.DynamicWriteMatrix2DCP",exectype="mem")
solve = externalFunction(Matrix[Double] A, Matrix[Double] y)
return (Matrix[Double] b)
-implemented in (classname="com.ibm.bi.dml.packagesupport.LinearSolverWrapperCP",exectype="mem")
+implemented in (classname="org.apache.sysml.packagesupport.LinearSolverWrapperCP",exectype="mem")
k = %numModels%;
m = -1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index e9b2f43..0f2c7aa 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -15,49 +15,49 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.io.IOException;
import java.util.ArrayList;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
index 7303fbf..5603484 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.stat;
+package org.apache.sysml.runtime.controlprogram.parfor.stat;
import java.io.IOException;
import java.util.StringTokenizer;
@@ -23,8 +23,8 @@ import java.util.StringTokenizer;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRConfigurationNames;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames;
/**
* Central place for analyzing and obtaining static infrastructure properties
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Stat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Stat.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Stat.java
index ff51907..913c780 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Stat.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Stat.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.stat;
+package org.apache.sysml.runtime.controlprogram.parfor.stat;
/**
* Collection of all ParFor statistic types.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
index bfd8819..378d48b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.stat;
+package org.apache.sysml.runtime.controlprogram.parfor.stat;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map.Entry;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
/**
* This singleton statistic monitor is used to consolidate all parfor runtime statistics.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Timing.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Timing.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Timing.java
index 6e95170..6e0dabd 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Timing.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/Timing.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.stat;
+package org.apache.sysml.runtime.controlprogram.parfor.stat;
/**
* Helper class for encapsulated time measurements.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/Cell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/Cell.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/Cell.java
index d283d94..e9208e4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/Cell.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/Cell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
/**
* Helper class for representing text cell and binary cell records in order to
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
index 90bee08..df22486 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDSequence.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDSequence.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDSequence.java
index 34fecf8..3e6a35c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDSequence.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDSequence.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
/**
* ID sequence for generating unique long identifiers with start 0 and increment 1.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableBlock.java
index 29e6127..660ab71 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
import java.io.DataInput;
import java.io.DataOutput;
@@ -24,8 +24,8 @@ import java.io.Serializable;
import org.apache.hadoop.io.Writable;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Custom writable for a pair of matrix indexes and matrix block
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableCell.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableCell.java
index 8b5fdd3..d0c48b8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableCell.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/PairWritableCell.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
import java.io.DataInput;
import java.io.DataOutput;
@@ -24,8 +24,8 @@ import java.io.Serializable;
import org.apache.hadoop.io.Writable;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Custom writable for a pair of matrix indexes and matrix cell
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/StagingFileUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/StagingFileUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/StagingFileUtils.java
index 1af077e..7b53c04 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/StagingFileUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/StagingFileUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.util;
+package org.apache.sysml.runtime.controlprogram.parfor.util;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -28,9 +28,9 @@ import java.io.OutputStreamWriter;
import java.util.HashMap;
import java.util.LinkedList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
public class StagingFileUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/And.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/And.java b/src/main/java/org/apache/sysml/runtime/functionobjects/And.java
index cf41848..4511acf 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/And.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/And.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
index ac09940..6dc516f 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.util.HashMap;
import org.apache.commons.math3.util.FastMath;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/CM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/CM.java b/src/main/java/org/apache/sysml/runtime/functionobjects/CM.java
index e21060e..203b7c6 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/CM.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/CM.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/COV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/COV.java b/src/main/java/org/apache/sysml/runtime/functionobjects/COV.java
index f89bbe6..1ab5312 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/COV.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/COV.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/CTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/CTable.java b/src/main/java/org/apache/sysml/runtime/functionobjects/CTable.java
index fb518f0..0e93a97 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/CTable.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/CTable.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.CTableMap;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.CTableMap;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/DiagIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/DiagIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/DiagIndex.java
index 957d422..a4bee9a 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/DiagIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/DiagIndex.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class DiagIndex extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Divide.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Divide.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Divide.java
index 53800e6..d62966b 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Divide.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Divide.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class Divide extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Equals.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Equals.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Equals.java
index 7adcd54..1b38f72 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Equals.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Equals.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class Equals extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/FileFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/FileFunction.java b/src/main/java/org/apache/sysml/runtime/functionobjects/FileFunction.java
index d621457..5a1d66b 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/FileFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/FileFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class FileFunction extends FunctionObject {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/FunctionObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/FunctionObject.java b/src/main/java/org/apache/sysml/runtime/functionobjects/FunctionObject.java
index 8613882..918798c 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/FunctionObject.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/FunctionObject.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.util.HashMap;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class FunctionObject
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThan.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThan.java b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThan.java
index 5653feb..6ba59a0 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThan.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThan.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class GreaterThan extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThanEquals.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThanEquals.java b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThanEquals.java
index 5133b54..1df4331 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThanEquals.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/GreaterThanEquals.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class GreaterThanEquals extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/IndexFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/IndexFunction.java b/src/main/java/org/apache/sysml/runtime/functionobjects/IndexFunction.java
index 0e8e4a2..7939a60 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/IndexFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/IndexFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java b/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java
index 4e317ab..c4d6ba4 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class IntegerDivide extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanFunction.java b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanFunction.java
index 6ac0a7c..d1a6dd4 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanFunction.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlus.java b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlus.java
index df5997b..fafdce3 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlus.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlus.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
public class KahanPlus extends KahanFunction implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlusSq.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlusSq.java b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlusSq.java
index b02b8c2..0879df6 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlusSq.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/KahanPlusSq.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThan.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/LessThan.java b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThan.java
index 42e86a3..792d255 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/LessThan.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThan.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class LessThan extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThanEquals.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/LessThanEquals.java b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThanEquals.java
index ed02fef..3d97d7d 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/LessThanEquals.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/LessThanEquals.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class LessThanEquals extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
index f1477e7..4612bc7 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class MaxIndex extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Mean.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Mean.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Mean.java
index b800e75..39daab3 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Mean.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Mean.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
public class Mean extends ValueFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
index a415e76..13cb440 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class MinIndex extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Minus.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus.java
index b9a9236..64946a1 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Minus.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus1Multiply.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Minus1Multiply.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus1Multiply.java
index 6d36a7e..4041dda 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Minus1Multiply.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Minus1Multiply.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/MinusNz.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/MinusNz.java b/src/main/java/org/apache/sysml/runtime/functionobjects/MinusNz.java
index 6cef0d1..9f8346d 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/MinusNz.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/MinusNz.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Modulus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Modulus.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Modulus.java
index 6840bef..f7d9281 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Modulus.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Modulus.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
/**
* Integer modulus, where we adhere to the defined R semantics:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply.java
index dc80be4..c9bbf40 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply2.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply2.java
index 01af490..2e5c04c 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply2.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Multiply2.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class Multiply2 extends ValueFunction
{
[35/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/com/ibm/bi/dml/hops/ipa/InterProceduralAnalysis.java
deleted file mode 100644
index 744c22f..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/ipa/InterProceduralAnalysis.java
+++ /dev/null
@@ -1,1248 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.ipa;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.udf.lib.DeNaNWrapper;
-import com.ibm.bi.dml.udf.lib.DeNegInfinityWrapper;
-import com.ibm.bi.dml.udf.lib.DynamicReadMatrixCP;
-import com.ibm.bi.dml.udf.lib.DynamicReadMatrixRcCP;
-import com.ibm.bi.dml.udf.lib.OrderWrapper;
-
-/**
- * This Inter Procedural Analysis (IPA) serves two major purposes:
- * 1) Inter-Procedure Analysis: propagate statistics from calling program into
- * functions and back into main program. This is done recursively for nested
- * function invocations.
- * 2) Intra-Procedural Analysis: propagate statistics across hop dags of subsequent
- * statement blocks in order to allow chained function calls and reasoning about
- * changing sparsity etc (that requires the rewritten hops dag as input). This
- * also includes control-flow aware propagation of size and sparsity. Furthermore,
- * it also serves as a second constant propagation pass.
- *
- * In general, the basic concepts of IPA are as follows and all places that deal with
- * statistic propagation should adhere to that:
- * * Rule 1: Exact size propagation: Since the dimension information are sometimes used
- * for specific lops construction (e.g., in append) and rewrites, we cannot propagate worst-case
- * estimates but only exact information; otherwise size must be unknown.
- * * Rule 2: Dimension information and sparsity are handled separately, i.e., if an updated
- * variable has changing sparsity but constant dimensions, its dimensions are known but
- * sparsity unknown.
- *
- * More specifically, those two rules are currently realized as follows:
- * * Statistics propagation is applied for DML-bodied functions that are invoked exactly once.
- * This ensures that we can savely propagate exact information into this function.
- * If ALLOW_MULTIPLE_FUNCTION_CALLS is enabled we treat multiple calls with the same sizes
- * as one call and hence, propagate those statistics into the function as well.
- * * Output size inference happens for DML-bodied functions that are invoked exactly once
- * and for external functions that are known in advance (see UDFs in com.ibm.bi.dml.udf).
- * * Size propagation across DAGs requires control flow awareness:
- * - Generic statement blocks: updated variables -> old stats in; new stats out
- * - While/for statement blocks: updated variables -> old stats in/out if loop insensitive; otherwise unknown
- * - If statement blocks: updated variables -> old stats in; new stats out if branch-insensitive
- *
- *
- */
-public class InterProceduralAnalysis
-{
-
- private static final boolean LDEBUG = false; //internal local debug level
- private static final Log LOG = LogFactory.getLog(InterProceduralAnalysis.class.getName());
-
- //internal configuration parameters
- private static final boolean INTRA_PROCEDURAL_ANALYSIS = true; //propagate statistics across statement blocks (main/functions)
- private static final boolean PROPAGATE_KNOWN_UDF_STATISTICS = true; //propagate statistics for known external functions
- private static final boolean ALLOW_MULTIPLE_FUNCTION_CALLS = true; //propagate consistent statistics from multiple calls
- private static final boolean REMOVE_UNUSED_FUNCTIONS = true; //remove unused functions (inlined or never called)
- private static final boolean FLAG_FUNCTION_RECOMPILE_ONCE = true; //flag functions which require recompilation inside a loop for full function recompile
- private static final boolean REMOVE_UNNECESSARY_CHECKPOINTS = true; //remove unnecessary checkpoints (unconditionally overwritten intermediates)
- private static final boolean REMOVE_CONSTANT_BINARY_OPS = true; //remove constant binary operations (e.g., X*ones, where ones=matrix(1,...))
-
- static {
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.parser.InterProceduralAnalysis")
- .setLevel((Level) Level.DEBUG);
- }
- }
-
- public InterProceduralAnalysis()
- {
- //do nothing
- }
-
- /**
- * Public interface of IPA - everything else is meant for internal use only.
- *
- * @param dmlt
- * @param dmlp
- * @throws HopsException
- * @throws ParseException
- * @throws LanguageException
- */
- public void analyzeProgram( DMLProgram dmlp )
- throws HopsException, ParseException, LanguageException
- {
- //step 1: get candidates for statistics propagation into functions (if required)
- Map<String, Integer> fcandCounts = new HashMap<String, Integer>();
- Map<String, FunctionOp> fcandHops = new HashMap<String, FunctionOp>();
- Map<String, Set<Long>> fcandSafeNNZ = new HashMap<String, Set<Long>>();
- Set<String> allFCandKeys = new HashSet<String>();
- if( dmlp.getFunctionStatementBlocks().size() > 0 )
- {
- for ( StatementBlock sb : dmlp.getStatementBlocks() ) //get candidates (over entire program)
- getFunctionCandidatesForStatisticPropagation( sb, fcandCounts, fcandHops );
- allFCandKeys.addAll(fcandCounts.keySet()); //cp before pruning
- pruneFunctionCandidatesForStatisticPropagation( fcandCounts, fcandHops );
- determineFunctionCandidatesNNZPropagation( fcandHops, fcandSafeNNZ );
- DMLTranslator.resetHopsDAGVisitStatus( dmlp );
- }
-
- if( !fcandCounts.isEmpty() || INTRA_PROCEDURAL_ANALYSIS ) {
- //step 2: propagate statistics into functions and across DAGs
- //(callVars used to chain outputs/inputs of multiple functions calls)
- LocalVariableMap callVars = new LocalVariableMap();
- for ( StatementBlock sb : dmlp.getStatementBlocks() ) //propagate stats into candidates
- propagateStatisticsAcrossBlock( sb, fcandCounts.keySet(), callVars, fcandSafeNNZ, new HashSet<String>() );
- }
-
- //step 3: remove unused functions (e.g., inlined or never called)
- if( REMOVE_UNUSED_FUNCTIONS ) {
- removeUnusedFunctions( dmlp, allFCandKeys );
- }
-
- //step 4: flag functions with loops for 'recompile-on-entry'
- if( FLAG_FUNCTION_RECOMPILE_ONCE ) {
- flagFunctionsForRecompileOnce( dmlp );
- }
-
- //step 5: set global data flow properties
- if( REMOVE_UNNECESSARY_CHECKPOINTS
- && OptimizerUtils.isSparkExecutionMode() )
- {
- removeUnnecessaryCheckpoints(dmlp);
- }
-
- //step 6: remove constant binary ops
- if( REMOVE_CONSTANT_BINARY_OPS ) {
- removeConstantBinaryOps(dmlp);
- }
- }
-
- /**
- *
- * @param sb
- * @return
- * @throws ParseException
- * @throws HopsException
- */
- public Set<String> analyzeSubProgram( StatementBlock sb )
- throws HopsException, ParseException
- {
- DMLTranslator.resetHopsDAGVisitStatus(sb);
-
- //step 1: get candidates for statistics propagation into functions (if required)
- Map<String, Integer> fcandCounts = new HashMap<String, Integer>();
- Map<String, FunctionOp> fcandHops = new HashMap<String, FunctionOp>();
- Map<String, Set<Long>> fcandSafeNNZ = new HashMap<String, Set<Long>>();
- Set<String> allFCandKeys = new HashSet<String>();
- getFunctionCandidatesForStatisticPropagation( sb, fcandCounts, fcandHops );
- allFCandKeys.addAll(fcandCounts.keySet()); //cp before pruning
- pruneFunctionCandidatesForStatisticPropagation( fcandCounts, fcandHops );
- determineFunctionCandidatesNNZPropagation( fcandHops, fcandSafeNNZ );
- DMLTranslator.resetHopsDAGVisitStatus( sb );
-
- if( !fcandCounts.isEmpty() ) {
- //step 2: propagate statistics into functions and across DAGs
- //(callVars used to chain outputs/inputs of multiple functions calls)
- LocalVariableMap callVars = new LocalVariableMap();
- propagateStatisticsAcrossBlock( sb, fcandCounts.keySet(), callVars, fcandSafeNNZ, new HashSet<String>() );
- }
-
- return fcandCounts.keySet();
- }
-
-
- /////////////////////////////
- // GET FUNCTION CANDIDATES
- //////
-
- /**
- *
- * @param sb
- * @param fcand
- * @throws HopsException
- * @throws ParseException
- */
- private void getFunctionCandidatesForStatisticPropagation( StatementBlock sb, Map<String, Integer> fcandCounts, Map<String, FunctionOp> fcandHops )
- throws HopsException, ParseException
- {
- if (sb instanceof FunctionStatementBlock)
- {
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- for (StatementBlock sbi : fstmt.getBody())
- getFunctionCandidatesForStatisticPropagation(sbi, fcandCounts, fcandHops);
- }
- else if (sb instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- for (StatementBlock sbi : wstmt.getBody())
- getFunctionCandidatesForStatisticPropagation(sbi, fcandCounts, fcandHops);
- }
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- for (StatementBlock sbi : istmt.getIfBody())
- getFunctionCandidatesForStatisticPropagation(sbi, fcandCounts, fcandHops);
- for (StatementBlock sbi : istmt.getElseBody())
- getFunctionCandidatesForStatisticPropagation(sbi, fcandCounts, fcandHops);
- }
- else if (sb instanceof ForStatementBlock) //incl parfor
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- for (StatementBlock sbi : fstmt.getBody())
- getFunctionCandidatesForStatisticPropagation(sbi, fcandCounts, fcandHops);
- }
- else //generic (last-level)
- {
- ArrayList<Hop> roots = sb.get_hops();
- if( roots != null ) //empty statement blocks
- for( Hop root : roots )
- getFunctionCandidatesForStatisticPropagation(sb.getDMLProg(), root, fcandCounts, fcandHops);
- }
- }
-
- /**
- *
- * @param prog
- * @param hop
- * @param fcand
- * @throws HopsException
- * @throws ParseException
- */
- private void getFunctionCandidatesForStatisticPropagation(DMLProgram prog, Hop hop, Map<String, Integer> fcandCounts, Map<String, FunctionOp> fcandHops )
- throws HopsException, ParseException
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- if( hop instanceof FunctionOp && !((FunctionOp)hop).getFunctionNamespace().equals(DMLProgram.INTERNAL_NAMESPACE) )
- {
- //maintain counters and investigate functions if not seen so far
- FunctionOp fop = (FunctionOp) hop;
- String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
-
- if( fcandCounts.containsKey(fkey) ) {
- if( ALLOW_MULTIPLE_FUNCTION_CALLS )
- {
- //compare input matrix characteristics for both function calls
- //(if unknown or difference: maintain counter - this function is no candidate)
- boolean consistent = true;
- FunctionOp efop = fcandHops.get(fkey);
- int numInputs = efop.getInput().size();
- for( int i=0; i<numInputs; i++ )
- {
- Hop h1 = efop.getInput().get(i);
- Hop h2 = fop.getInput().get(i);
- //check matrix and scalar sizes (if known dims, nnz known/unknown,
- // safeness of nnz propagation, determined later per input)
- consistent &= (h1.dimsKnown() && h2.dimsKnown()
- && h1.getDim1()==h2.getDim1()
- && h1.getDim2()==h2.getDim2()
- && h1.getNnz()==h2.getNnz() );
- //check literal values (equi value)
- if( h1 instanceof LiteralOp ){
- consistent &= (h2 instanceof LiteralOp
- && HopRewriteUtils.isEqualValue((LiteralOp)h1, (LiteralOp)h2));
- }
-
-
- }
-
- if( !consistent ) //if differences, do not propagate
- fcandCounts.put(fkey, fcandCounts.get(fkey)+1);
- }
- else
- {
- //maintain counter (this function is no candidate)
- fcandCounts.put(fkey, fcandCounts.get(fkey)+1);
- }
- }
- else { //first appearance
- fcandCounts.put(fkey, 1); //create a new count entry
- fcandHops.put(fkey, fop); //keep the function call hop
- FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
- getFunctionCandidatesForStatisticPropagation(fsb, fcandCounts, fcandHops);
- }
- }
-
- for( Hop c : hop.getInput() )
- getFunctionCandidatesForStatisticPropagation(prog, c, fcandCounts, fcandHops);
-
- hop.setVisited(VisitStatus.DONE);
- }
-
- /**
- *
- * @param fcand
- */
- private void pruneFunctionCandidatesForStatisticPropagation(Map<String, Integer> fcandCounts, Map<String, FunctionOp> fcandHops)
- {
- //debug input
- if( LOG.isDebugEnabled() )
- for( Entry<String,Integer> e : fcandCounts.entrySet() )
- {
- String key = e.getKey();
- Integer count = e.getValue();
- LOG.debug("IPA: FUNC statistic propagation candidate: "+key+", callCount="+count);
- }
-
- //materialize key set
- Set<String> tmp = new HashSet<String>(fcandCounts.keySet());
-
- //check and prune candidate list
- for( String key : tmp )
- {
- Integer cnt = fcandCounts.get(key);
- if( cnt != null && cnt > 1 ) //if multiple refs
- fcandCounts.remove(key);
- }
-
- //debug output
- if( LOG.isDebugEnabled() )
- for( String key : fcandCounts.keySet() )
- {
- LOG.debug("IPA: FUNC statistic propagation candidate (after pruning): "+key);
- }
- }
-
- /////////////////////////////
- // DETERMINE NNZ PROPAGATE SAFENESS
- //////
-
- /**
- * Populates fcandSafeNNZ with all <functionKey,hopID> pairs where it is safe to
- * propagate nnz into the function.
- *
- * @param fcandHops
- * @param fcandSafeNNZ
- */
- private void determineFunctionCandidatesNNZPropagation(Map<String, FunctionOp> fcandHops, Map<String, Set<Long>> fcandSafeNNZ)
- {
- //for all function candidates
- for( Entry<String, FunctionOp> e : fcandHops.entrySet() )
- {
- String fKey = e.getKey();
- FunctionOp fop = e.getValue();
- HashSet<Long> tmp = new HashSet<Long>();
-
- //for all inputs of this function call
- for( Hop input : fop.getInput() )
- {
- //if nnz known it is safe to propagate those nnz because for multiple calls
- //we checked of equivalence and hence all calls have the same nnz
- if( input.getNnz()>=0 )
- tmp.add(input.getHopID());
- }
-
- fcandSafeNNZ.put(fKey, tmp);
- }
- }
-
- /////////////////////////////
- // INTRA-PROCEDURE ANALYSIS
- //////
-
- /**
- *
- * @param sb
- * @param fcand
- * @throws HopsException
- * @throws ParseException
- * @throws CloneNotSupportedException
- */
- private void propagateStatisticsAcrossBlock( StatementBlock sb, Set<String> fcand, LocalVariableMap callVars, Map<String, Set<Long>> fcandSafeNNZ, Set<String> fnStack )
- throws HopsException, ParseException
- {
- if (sb instanceof FunctionStatementBlock)
- {
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- for (StatementBlock sbi : fstmt.getBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- }
- else if (sb instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- //old stats into predicate
- propagateStatisticsAcrossPredicateDAG(wsb.getPredicateHops(), callVars);
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, wsb);
- //check and propagate stats into body
- LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
- for (StatementBlock sbi : wstmt.getBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- if( Recompiler.reconcileUpdatedCallVarsLoops(oldCallVars, callVars, wsb) ){ //second pass if required
- propagateStatisticsAcrossPredicateDAG(wsb.getPredicateHops(), callVars);
- for (StatementBlock sbi : wstmt.getBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- }
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, sb);
- }
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- //old stats into predicate
- propagateStatisticsAcrossPredicateDAG(isb.getPredicateHops(), callVars);
- //check and propagate stats into body
- LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
- LocalVariableMap callVarsElse = (LocalVariableMap) callVars.clone();
- for (StatementBlock sbi : istmt.getIfBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- for (StatementBlock sbi : istmt.getElseBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVarsElse, fcandSafeNNZ, fnStack);
- callVars = Recompiler.reconcileUpdatedCallVarsIf(oldCallVars, callVars, callVarsElse, isb);
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, sb);
- }
- else if (sb instanceof ForStatementBlock) //incl parfor
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- //old stats into predicate
- propagateStatisticsAcrossPredicateDAG(fsb.getFromHops(), callVars);
- propagateStatisticsAcrossPredicateDAG(fsb.getToHops(), callVars);
- propagateStatisticsAcrossPredicateDAG(fsb.getIncrementHops(), callVars);
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, fsb);
- //check and propagate stats into body
- LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
- for (StatementBlock sbi : fstmt.getBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- if( Recompiler.reconcileUpdatedCallVarsLoops(oldCallVars, callVars, fsb) )
- for (StatementBlock sbi : fstmt.getBody())
- propagateStatisticsAcrossBlock(sbi, fcand, callVars, fcandSafeNNZ, fnStack);
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, sb);
- }
- else //generic (last-level)
- {
- //remove updated constant scalars
- Recompiler.removeUpdatedScalars(callVars, sb);
- //old stats in, new stats out if updated
- ArrayList<Hop> roots = sb.get_hops();
- DMLProgram prog = sb.getDMLProg();
- //refresh stats across dag
- Hop.resetVisitStatus(roots);
- propagateStatisticsAcrossDAG(roots, callVars);
- //propagate stats into function calls
- Hop.resetVisitStatus(roots);
- propagateStatisticsIntoFunctions(prog, roots, fcand, callVars, fcandSafeNNZ, fnStack);
- }
- }
-
-
- /**
- *
- * @param root
- * @param vars
- * @throws HopsException
- */
- private void propagateStatisticsAcrossPredicateDAG( Hop root, LocalVariableMap vars )
- throws HopsException
- {
- if( root == null )
- return;
-
- //reset visit status because potentially called multiple times
- root.resetVisitStatus();
-
- try
- {
- Recompiler.rUpdateStatistics( root, vars );
-
- //note: for predicates no output statistics
- //Recompiler.extractDAGOutputStatistics(root, vars);
- }
- catch(Exception ex)
- {
- throw new HopsException("Failed to update Hop DAG statistics.", ex);
- }
- }
-
-
- /**
- *
- * @param roots
- * @param vars
- * @throws HopsException
- */
- private void propagateStatisticsAcrossDAG( ArrayList<Hop> roots, LocalVariableMap vars )
- throws HopsException
- {
- if( roots == null )
- return;
-
- try
- {
- //update DAG statistics from leafs to roots
- for( Hop hop : roots )
- Recompiler.rUpdateStatistics( hop, vars );
-
- //extract statistics from roots
- Recompiler.extractDAGOutputStatistics(roots, vars, true);
- }
- catch( Exception ex )
- {
- throw new HopsException("Failed to update Hop DAG statistics.", ex);
- }
- }
-
-
- /////////////////////////////
- // INTER-PROCEDURE ANALYIS
- //////
-
-
- /**
- *
- * @param prog
- * @param hop
- * @param fcand
- * @param callVars
- * @throws HopsException
- * @throws ParseException
- */
- private void propagateStatisticsIntoFunctions(DMLProgram prog, ArrayList<Hop> roots, Set<String> fcand, LocalVariableMap callVars, Map<String, Set<Long>> fcandSafeNNZ, Set<String> fnStack )
- throws HopsException, ParseException
- {
- for( Hop root : roots )
- propagateStatisticsIntoFunctions(prog, root, fcand, callVars, fcandSafeNNZ, fnStack);
- }
-
-
- /**
- *
- * @param prog
- * @param hop
- * @param fcand
- * @throws HopsException
- * @throws ParseException
- */
- private void propagateStatisticsIntoFunctions(DMLProgram prog, Hop hop, Set<String> fcand, LocalVariableMap callVars, Map<String, Set<Long>> fcandSafeNNZ, Set<String> fnStack )
- throws HopsException, ParseException
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- for( Hop c : hop.getInput() )
- propagateStatisticsIntoFunctions(prog, c, fcand, callVars, fcandSafeNNZ, fnStack);
-
- if( hop instanceof FunctionOp )
- {
- //maintain counters and investigate functions if not seen so far
- FunctionOp fop = (FunctionOp) hop;
- String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
-
- if( fop.getFunctionType() == FunctionType.DML )
- {
- FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
-
- if( fcand.contains(fkey) &&
- !fnStack.contains(fkey) ) //prevent recursion
- {
- //maintain function call stack
- fnStack.add(fkey);
-
- //create mapping and populate symbol table for refresh
- LocalVariableMap tmpVars = new LocalVariableMap();
- populateLocalVariableMapForFunctionCall( fstmt, fop, tmpVars, fcandSafeNNZ.get(fkey) );
-
- //recursively propagate statistics
- propagateStatisticsAcrossBlock(fsb, fcand, tmpVars, fcandSafeNNZ, fnStack);
-
- //extract vars from symbol table, re-map and refresh main program
- extractFunctionCallReturnStatistics(fstmt, fop, tmpVars, callVars, true);
-
- //maintain function call stack
- fnStack.remove(fkey);
- }
- else
- {
- extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
- }
- }
- else if ( fop.getFunctionType() == FunctionType.EXTERNAL_FILE
- || fop.getFunctionType() == FunctionType.EXTERNAL_MEM )
- {
- //infer output size for known external functions
- FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
- ExternalFunctionStatement fstmt = (ExternalFunctionStatement) fsb.getStatement(0);
- if( PROPAGATE_KNOWN_UDF_STATISTICS )
- extractExternalFunctionCallReturnStatistics(fstmt, fop, callVars);
- else
- extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
- }
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-
-
- /**
- *
- * @param fstmt
- * @param fop
- * @param vars
- * @throws HopsException
- */
- private void populateLocalVariableMapForFunctionCall( FunctionStatement fstmt, FunctionOp fop, LocalVariableMap vars, Set<Long> inputSafeNNZ )
- throws HopsException
- {
- ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
- ArrayList<Hop> inputOps = fop.getInput();
-
- for( int i=0; i<inputVars.size(); i++ )
- {
- //create mapping between input hops and vars
- DataIdentifier dat = inputVars.get(i);
- Hop input = inputOps.get(i);
-
- if( input.getDataType()==DataType.MATRIX )
- {
- //propagate matrix characteristics
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
- MatrixCharacteristics mc = new MatrixCharacteristics(
- input.getDim1(), input.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
- inputSafeNNZ.contains(input.getHopID())?input.getNnz():-1 );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
- mo.setMetaData(meta);
- vars.put(dat.getName(), mo);
- }
- else if( input.getDataType()==DataType.SCALAR
- && input instanceof LiteralOp )
- {
- //propagate literal scalars into functions
- LiteralOp lit = (LiteralOp)input;
- ScalarObject scalar = null;
- switch(input.getValueType())
- {
- case DOUBLE: scalar = new DoubleObject(lit.getDoubleValue()); break;
- case INT: scalar = new IntObject(lit.getLongValue()); break;
- case BOOLEAN: scalar = new BooleanObject(lit.getBooleanValue()); break;
- case STRING: scalar = new StringObject(lit.getStringValue()); break;
- default:
- //do nothing
- }
- vars.put(dat.getName(), scalar);
- }
-
- }
- }
-
- /**
- *
- * @param fstmt
- * @param fop
- * @param tmpVars
- * @param callVars
- * @param overwrite
- * @throws HopsException
- */
- private void extractFunctionCallReturnStatistics( FunctionStatement fstmt, FunctionOp fop, LocalVariableMap tmpVars, LocalVariableMap callVars, boolean overwrite )
- throws HopsException
- {
- ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
- String[] outputVars = fop.getOutputVariableNames();
- String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
-
- try
- {
- for( int i=0; i<foutputOps.size(); i++ )
- {
- DataIdentifier di = foutputOps.get(i);
- String fvarname = di.getName(); //name in function signature
- String pvarname = outputVars[i]; //name in calling program
-
- if( di.getDataType()==DataType.MATRIX && tmpVars.keySet().contains(fvarname) )
- {
- MatrixObject moIn = (MatrixObject) tmpVars.get(fvarname);
-
- if( !callVars.keySet().contains(pvarname) || overwrite ) //not existing so far
- {
- MatrixObject moOut = createOutputMatrix(moIn.getNumRows(), moIn.getNumColumns(), moIn.getNnz());
- callVars.put(pvarname, moOut);
- }
- else //already existing: take largest
- {
- Data dat = callVars.get(pvarname);
- if( dat instanceof MatrixObject )
- {
- MatrixObject moOut = (MatrixObject)dat;
- MatrixCharacteristics mc = moOut.getMatrixCharacteristics();
- if( OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros()>0)?((double)mc.getNonZeros())/mc.getRows()/mc.getCols():1.0)
- < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns()) )
- {
- //update statistics if necessary
- mc.setDimension(moIn.getNumRows(), moIn.getNumColumns());
- mc.setNonZeros(moIn.getNnz());
- }
- }
-
- }
- }
- }
- }
- catch( Exception ex )
- {
- throw new HopsException( "Failed to extract output statistics of function "+fkey+".", ex);
- }
- }
-
- /**
- *
- * @param fstmt
- * @param fop
- * @param callVars
- * @throws HopsException
- */
- private void extractFunctionCallUnknownReturnStatistics( FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars )
- throws HopsException
- {
- ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
- String[] outputVars = fop.getOutputVariableNames();
- String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
-
- try
- {
- for( int i=0; i<foutputOps.size(); i++ )
- {
- DataIdentifier di = foutputOps.get(i);
- String pvarname = outputVars[i]; //name in calling program
-
- if( di.getDataType()==DataType.MATRIX )
- {
- MatrixObject moOut = createOutputMatrix(-1, -1, -1);
- callVars.put(pvarname, moOut);
- }
- }
- }
- catch( Exception ex )
- {
- throw new HopsException( "Failed to extract output statistics of function "+fkey+".", ex);
- }
- }
-
- /**
- *
- * @param fstmt
- * @param fop
- * @param callVars
- * @throws HopsException
- */
- private void extractExternalFunctionCallReturnStatistics( ExternalFunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars )
- throws HopsException
- {
- String className = fstmt.getOtherParams().get(ExternalFunctionStatement.CLASS_NAME);
-
- if( className.equals(OrderWrapper.class.getName())
- || className.equals(DeNaNWrapper.class.getCanonicalName())
- || className.equals(DeNegInfinityWrapper.class.getCanonicalName()) )
- {
- Hop input = fop.getInput().get(0);
- long lnnz = className.equals(OrderWrapper.class.getName()) ? input.getNnz() : -1;
- MatrixObject moOut = createOutputMatrix(input.getDim1(), input.getDim2(),lnnz);
- callVars.put(fop.getOutputVariableNames()[0], moOut);
- }
- else if( className.equals("com.ibm.bi.dml.udf.lib.EigenWrapper") )
- //else if( className.equals(EigenWrapper.class.getName()) ) //string ref for build flexibility
- {
- Hop input = fop.getInput().get(0);
- callVars.put(fop.getOutputVariableNames()[0], createOutputMatrix(input.getDim1(), 1, -1));
- callVars.put(fop.getOutputVariableNames()[1], createOutputMatrix(input.getDim1(), input.getDim1(),-1));
- }
- else if( className.equals("com.ibm.bi.dml.udf.lib.LinearSolverWrapperCP") )
- //else if( className.equals(LinearSolverWrapperCP.class.getName()) ) //string ref for build flexibility
- {
- Hop input = fop.getInput().get(1);
- callVars.put(fop.getOutputVariableNames()[0], createOutputMatrix(input.getDim1(), 1, -1));
- }
- else if( className.equals(DynamicReadMatrixCP.class.getName())
- || className.equals(DynamicReadMatrixRcCP.class.getName()) )
- {
- Hop input1 = fop.getInput().get(1); //rows
- Hop input2 = fop.getInput().get(2); //cols
- if( input1 instanceof LiteralOp && input2 instanceof LiteralOp )
- callVars.put(fop.getOutputVariableNames()[0], createOutputMatrix(((LiteralOp)input1).getLongValue(),
- ((LiteralOp)input2).getLongValue(),-1));
- }
- else
- {
- extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
- }
- }
-
- /**
- *
- * @param dim1
- * @param dim2
- * @param nnz
- * @return
- */
- private MatrixObject createOutputMatrix( long dim1, long dim2, long nnz )
- {
- MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
- MatrixCharacteristics mc = new MatrixCharacteristics(
- dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
- nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
- moOut.setMetaData(meta);
-
- return moOut;
- }
-
- /////////////////////////////
- // REMOVE UNUSED FUNCTIONS
- //////
-
- /**
- *
- * @param dmlp
- * @param fcandKeys
- * @throws LanguageException
- */
- public void removeUnusedFunctions( DMLProgram dmlp, Set<String> fcandKeys )
- throws LanguageException
- {
- Set<String> fnamespaces = dmlp.getNamespaces().keySet();
- for( String fnspace : fnamespaces )
- {
- HashMap<String, FunctionStatementBlock> fsbs = dmlp.getFunctionStatementBlocks(fnspace);
- Iterator<Entry<String, FunctionStatementBlock>> iter = fsbs.entrySet().iterator();
- while( iter.hasNext() )
- {
- Entry<String, FunctionStatementBlock> e = iter.next();
- String fname = e.getKey();
- String fKey = DMLProgram.constructFunctionKey(fnspace, fname);
- //probe function candidates, remove if no candidate
- if( !fcandKeys.contains(fKey) )
- iter.remove();
- }
- }
- }
-
-
- /////////////////////////////
- // FLAG FUNCTIONS FOR RECOMPILE_ONCE
- //////
-
- /**
- * TODO call it after construct lops
- *
- * @param dmlp
- * @throws LanguageException
- */
- public void flagFunctionsForRecompileOnce( DMLProgram dmlp )
- throws LanguageException
- {
- for (String namespaceKey : dmlp.getNamespaces().keySet())
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet())
- {
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- if( rFlagFunctionForRecompileOnce( fsblock, false ) )
- {
- fsblock.setRecompileOnce( true );
- LOG.debug("IPA: FUNC flagged for recompile-once: " + DMLProgram.constructFunctionKey(namespaceKey, fname));
- }
- }
- }
-
- /**
- * Returns true if this statementblock requires recompilation inside a
- * loop statement block.
- *
- *
- *
- * @param sb
- */
- public boolean rFlagFunctionForRecompileOnce( StatementBlock sb, boolean inLoop )
- {
- boolean ret = false;
-
- if (sb instanceof FunctionStatementBlock)
- {
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- for( StatementBlock c : fstmt.getBody() )
- ret |= rFlagFunctionForRecompileOnce( c, inLoop );
- }
- else if (sb instanceof WhileStatementBlock)
- {
- //recompilation information not available at this point
- ret = true;
-
- /*
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- ret |= (inLoop && wsb.requiresPredicateRecompilation() );
- for( StatementBlock c : wstmt.getBody() )
- ret |= rFlagFunctionForRecompileOnce( c, true );
- */
- }
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- ret |= (inLoop && isb.requiresPredicateRecompilation() );
- for( StatementBlock c : istmt.getIfBody() )
- ret |= rFlagFunctionForRecompileOnce( c, inLoop );
- for( StatementBlock c : istmt.getElseBody() )
- ret |= rFlagFunctionForRecompileOnce( c, inLoop );
- }
- else if (sb instanceof ForStatementBlock)
- {
- //recompilation information not available at this point
- ret = true;
-
- /*
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- for( StatementBlock c : fstmt.getBody() )
- ret |= rFlagFunctionForRecompileOnce( c, true );
- */
- }
- else
- {
- ret |= ( inLoop && sb.requiresRecompilation() );
- }
-
- return ret;
- }
-
- /////////////////////////////
- // REMOVE UNNECESSARY CHECKPOINTS
- //////
-
- /**
- *
- * @param dmlp
- * @throws HopsException
- */
- private void removeUnnecessaryCheckpoints(DMLProgram dmlp)
- throws HopsException
- {
- //approach: scan over top-level program (guaranteed to be unconditional),
- //collect checkpoints; determine if used before update; remove first checkpoint
- //on second checkpoint if update in between and not used before update
-
- HashMap<String, Hop> chkpointCand = new HashMap<String, Hop>();
-
- for( StatementBlock sb : dmlp.getStatementBlocks() )
- {
- //prune candidates (used before updated)
- Set<String> cands = new HashSet<String>(chkpointCand.keySet());
- for( String cand : cands )
- if( sb.variablesRead().containsVariable(cand)
- && !sb.variablesUpdated().containsVariable(cand) )
- {
- //note: variableRead might include false positives due to meta
- //data operations like nrow(X) or operations removed by rewrites
- //double check hops on basic blocks; otherwise worst-case
- boolean skipRemove = false;
- if( sb.get_hops() !=null ) {
- Hop.resetVisitStatus(sb.get_hops());
- skipRemove = true;
- for( Hop root : sb.get_hops() )
- skipRemove &= !HopRewriteUtils.rContainsRead(root, cand, false);
- }
- if( !skipRemove )
- chkpointCand.remove(cand);
- }
-
- //prune candidates (updated in conditional control flow)
- Set<String> cands2 = new HashSet<String>(chkpointCand.keySet());
- if( sb instanceof IfStatementBlock || sb instanceof WhileStatementBlock
- || sb instanceof ForStatementBlock )
- {
- for( String cand : cands2 )
- if( sb.variablesUpdated().containsVariable(cand) ) {
- chkpointCand.remove(cand);
- }
- }
- //prune candidates (updated w/ multiple reads)
- else
- {
- for( String cand : cands2 )
- if( sb.variablesUpdated().containsVariable(cand) && sb.get_hops() != null)
- {
- ArrayList<Hop> hops = sb.get_hops();
- Hop.resetVisitStatus(hops);
- for( Hop root : hops )
- if( root.getName().equals(cand) &&
- !HopRewriteUtils.rHasSimpleReadChain(root, cand) ) {
- chkpointCand.remove(cand);
- }
- }
- }
-
- //collect checkpoints and remove unnecessary checkpoints
- ArrayList<Hop> tmp = collectCheckpoints(sb.get_hops());
- for( Hop chkpoint : tmp ) {
- if( chkpointCand.containsKey(chkpoint.getName()) ) {
- chkpointCand.get(chkpoint.getName()).setRequiresCheckpoint(false);
- }
- chkpointCand.put(chkpoint.getName(), chkpoint);
- }
-
- }
- }
-
- /**
- *
- * @param roots
- * @return
- */
- private ArrayList<Hop> collectCheckpoints(ArrayList<Hop> roots)
- {
- ArrayList<Hop> ret = new ArrayList<Hop>();
- if( roots != null ) {
- Hop.resetVisitStatus(roots);
- for( Hop root : roots )
- rCollectCheckpoints(root, ret);
- }
-
- return ret;
- }
-
- /**
- *
- * @param hop
- * @param checkpoints
- */
- private void rCollectCheckpoints(Hop hop, ArrayList<Hop> checkpoints)
- {
- if( hop.getVisited()==VisitStatus.DONE )
- return;
-
- //handle leaf node for variable (checkpoint directly bound
- //to logical variable name and not used)
- if( hop.requiresCheckpoint() && hop.getParent().size()==1
- && hop.getParent().get(0) instanceof DataOp
- && ((DataOp)hop.getParent().get(0)).getDataOpType()==DataOpTypes.TRANSIENTWRITE)
- {
- checkpoints.add(hop);
- }
-
- //recursively process child nodes
- for( Hop c : hop.getInput() )
- rCollectCheckpoints(c, checkpoints);
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
- /////////////////////////////
- // REMOVE CONSTANT BINARY OPS
- //////
-
- /**
- *
- * @param dmlp
- * @throws HopsException
- */
- private void removeConstantBinaryOps(DMLProgram dmlp)
- throws HopsException
- {
- //approach: scan over top-level program (guaranteed to be unconditional),
- //collect ones=matrix(1,...); remove b(*)ones if not outer operation
- HashMap<String, Hop> mOnes = new HashMap<String, Hop>();
-
- for( StatementBlock sb : dmlp.getStatementBlocks() )
- {
- //pruning updated variables
- for( String var : sb.variablesUpdated().getVariableNames() )
- if( mOnes.containsKey( var ) )
- mOnes.remove( var );
-
- //replace constant binary ops
- if( !mOnes.isEmpty() )
- rRemoveConstantBinaryOp(sb, mOnes);
-
- //collect matrices of ones from last-level statement blocks
- if( !(sb instanceof IfStatementBlock || sb instanceof WhileStatementBlock
- || sb instanceof ForStatementBlock) )
- {
- collectMatrixOfOnes(sb.get_hops(), mOnes);
- }
- }
- }
-
- /**
- *
- * @param roots
- * @param mOnes
- */
- private void collectMatrixOfOnes(ArrayList<Hop> roots, HashMap<String,Hop> mOnes)
- {
- if( roots == null )
- return;
-
- for( Hop root : roots )
- if( root instanceof DataOp && ((DataOp)root).getDataOpType()==DataOpTypes.TRANSIENTWRITE
- && root.getInput().get(0) instanceof DataGenOp
- && ((DataGenOp)root.getInput().get(0)).hasConstantValue(1.0))
- {
- mOnes.put(root.getName(),root.getInput().get(0));
- }
- }
-
- /**
- *
- * @param sb
- * @param mOnes
- * @throws HopsException
- */
- private void rRemoveConstantBinaryOp(StatementBlock sb, HashMap<String,Hop> mOnes)
- throws HopsException
- {
- if( sb instanceof IfStatementBlock )
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement)isb.getStatement(0);
- for( StatementBlock c : istmt.getIfBody() )
- rRemoveConstantBinaryOp(c, mOnes);
- if( istmt.getElseBody() != null )
- for( StatementBlock c : istmt.getElseBody() )
- rRemoveConstantBinaryOp(c, mOnes);
- }
- else if( sb instanceof WhileStatementBlock )
- {
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- for( StatementBlock c : wstmt.getBody() )
- rRemoveConstantBinaryOp(c, mOnes);
- }
- else if( sb instanceof ForStatementBlock )
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement)fsb.getStatement(0);
- for( StatementBlock c : fstmt.getBody() )
- rRemoveConstantBinaryOp(c, mOnes);
- }
- else
- {
- if( sb.get_hops() != null ){
- Hop.resetVisitStatus(sb.get_hops());
- for( Hop hop : sb.get_hops() )
- rRemoveConstantBinaryOp(hop, mOnes);
- }
- }
- }
-
- /**
- *
- * @param hop
- * @param mOnes
- */
- private void rRemoveConstantBinaryOp(Hop hop, HashMap<String,Hop> mOnes)
- {
- if( hop.getVisited()==VisitStatus.DONE )
- return;
-
- if( hop instanceof BinaryOp && ((BinaryOp)hop).getOp()==OpOp2.MULT
- && !((BinaryOp) hop).isOuterVectorOperator()
- && hop.getInput().get(0).getDataType()==DataType.MATRIX
- && hop.getInput().get(1) instanceof DataOp
- && mOnes.containsKey(hop.getInput().get(1).getName()) )
- {
- //replace matrix of ones with literal 1 (later on removed by
- //algebraic simplification rewrites; otherwise more complex
- //recursive processing of childs and rewiring required)
- HopRewriteUtils.removeChildReferenceByPos(hop, hop.getInput().get(1), 1);
- HopRewriteUtils.addChildReference(hop, new LiteralOp(1), 1);
- }
-
- //recursively process child nodes
- for( Hop c : hop.getInput() )
- rRemoveConstantBinaryOp(c, mOnes);
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/recompile/LiteralReplacement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/recompile/LiteralReplacement.java b/src/main/java/com/ibm/bi/dml/hops/recompile/LiteralReplacement.java
deleted file mode 100644
index 9610be3..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/recompile/LiteralReplacement.java
+++ /dev/null
@@ -1,538 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.recompile;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.utils.Statistics;
-
-public class LiteralReplacement
-{
-
- //internal configuration parameters
- private static final long REPLACE_LITERALS_MAX_MATRIX_SIZE = 1000000; //10^6 cells (8MB)
- private static final boolean REPORT_LITERAL_REPLACE_OPS_STATS = true;
-
- /**
- *
- * @param hop
- * @param vars
- * @throws DMLRuntimeException
- */
- protected static void rReplaceLiterals( Hop hop, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- if( hop.getInput() != null )
- {
- //indexed access to allow parent-child modifications
- for( int i=0; i<hop.getInput().size(); i++ )
- {
- Hop c = hop.getInput().get(i);
- Hop lit = null;
-
- //conditional apply of literal replacements
- lit = (lit==null) ? replaceLiteralScalarRead(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralValueTypeCastScalarRead(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralValueTypeCastLiteral(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralDataTypeCastMatrixRead(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralValueTypeCastRightIndexing(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralFullUnaryAggregate(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralFullUnaryAggregateRightIndexing(c, vars) : lit;
-
- //replace hop w/ literal on demand
- if( lit != null )
- {
- //replace hop c by literal, for all parents to prevent (1) missed opportunities
- //because hop c marked as visited, and (2) repeated evaluation of uagg ops
-
- if( c.getParent().size() > 1 ) { //multiple parents
- ArrayList<Hop> parents = new ArrayList<Hop>(c.getParent());
- for( Hop p : parents ) {
- int pos = HopRewriteUtils.getChildReferencePos(p, c);
- HopRewriteUtils.removeChildReferenceByPos(p, c, pos);
- HopRewriteUtils.addChildReference(p, lit, pos);
- }
- }
- else { //current hop is only parent
- HopRewriteUtils.removeChildReferenceByPos(hop, c, i);
- HopRewriteUtils.addChildReference(hop, lit, i);
- }
- }
- //recursively process children
- else
- {
- rReplaceLiterals(c, vars);
- }
- }
- }
-
- hop.setVisited(VisitStatus.DONE);
- }
-
-
- ///////////////////////////////
- // Literal replacement rules
- ///////////////////////////////
-
- /**
- *
- * @param c
- * @param vars
- * @return
- */
- private static LiteralOp replaceLiteralScalarRead(Hop c, LocalVariableMap vars)
- {
- LiteralOp ret = null;
-
- //scalar read - literal replacement
- if( c instanceof DataOp && ((DataOp)c).getDataOpType() != DataOpTypes.PERSISTENTREAD
- && c.getDataType()==DataType.SCALAR )
- {
- Data dat = vars.get(c.getName());
- if( dat != null ) //required for selective constant propagation
- {
- ScalarObject sdat = (ScalarObject)dat;
- switch( sdat.getValueType() ) {
- case INT:
- ret = new LiteralOp(sdat.getLongValue());
- break;
- case DOUBLE:
- ret = new LiteralOp(sdat.getDoubleValue());
- break;
- case BOOLEAN:
- ret = new LiteralOp(sdat.getBooleanValue());
- break;
- default:
- //otherwise: do nothing
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- */
- private static LiteralOp replaceLiteralValueTypeCastScalarRead( Hop c, LocalVariableMap vars )
- {
- LiteralOp ret = null;
-
- //as.double/as.integer/as.boolean over scalar read - literal replacement
- if( c instanceof UnaryOp && (((UnaryOp)c).getOp() == OpOp1.CAST_AS_DOUBLE
- || ((UnaryOp)c).getOp() == OpOp1.CAST_AS_INT || ((UnaryOp)c).getOp() == OpOp1.CAST_AS_BOOLEAN )
- && c.getInput().get(0) instanceof DataOp && c.getDataType()==DataType.SCALAR )
- {
- Data dat = vars.get(c.getInput().get(0).getName());
- if( dat != null ) //required for selective constant propagation
- {
- ScalarObject sdat = (ScalarObject)dat;
- UnaryOp cast = (UnaryOp) c;
- switch( cast.getOp() ) {
- case CAST_AS_INT:
- ret = new LiteralOp(sdat.getLongValue());
- break;
- case CAST_AS_DOUBLE:
- ret = new LiteralOp(sdat.getDoubleValue());
- break;
- case CAST_AS_BOOLEAN:
- ret = new LiteralOp(sdat.getBooleanValue());
- break;
- default:
- //otherwise: do nothing
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static LiteralOp replaceLiteralValueTypeCastLiteral( Hop c, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- LiteralOp ret = null;
-
- //as.double/as.integer/as.boolean over scalar literal (potentially created by other replacement
- //rewrite in same dag) - literal replacement
- if( c instanceof UnaryOp && (((UnaryOp)c).getOp() == OpOp1.CAST_AS_DOUBLE
- || ((UnaryOp)c).getOp() == OpOp1.CAST_AS_INT || ((UnaryOp)c).getOp() == OpOp1.CAST_AS_BOOLEAN )
- && c.getInput().get(0) instanceof LiteralOp )
- {
- LiteralOp sdat = (LiteralOp)c.getInput().get(0);
- UnaryOp cast = (UnaryOp) c;
- try
- {
- switch( cast.getOp() ) {
- case CAST_AS_INT:
- long ival = HopRewriteUtils.getIntValue(sdat);
- ret = new LiteralOp(ival);
- break;
- case CAST_AS_DOUBLE:
- double dval = HopRewriteUtils.getDoubleValue(sdat);
- ret = new LiteralOp(dval);
- break;
- case CAST_AS_BOOLEAN:
- boolean bval = HopRewriteUtils.getBooleanValue(sdat);
- ret = new LiteralOp(bval);
- break;
- default:
- //otherwise: do nothing
- }
- }
- catch(HopsException ex) {
- throw new DMLRuntimeException(ex);
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static LiteralOp replaceLiteralDataTypeCastMatrixRead( Hop c, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- LiteralOp ret = null;
-
- //as.scalar/matrix read - literal replacement
- if( c instanceof UnaryOp && ((UnaryOp)c).getOp() == OpOp1.CAST_AS_SCALAR
- && c.getInput().get(0) instanceof DataOp )
- {
- Data dat = vars.get(c.getInput().get(0).getName());
- if( dat != null ) //required for selective constant propagation
- {
- //cast as scalar (see VariableCPInstruction)
- MatrixObject mo = (MatrixObject)dat;
- MatrixBlock mBlock = mo.acquireRead();
- if( mBlock.getNumRows()!=1 || mBlock.getNumColumns()!=1 )
- throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix of dimension ("+mBlock.getNumRows()+" x "+mBlock.getNumColumns()+") to scalar.");
- double value = mBlock.getValue(0,0);
- mo.release();
-
- //literal substitution (always double)
- ret = new LiteralOp(value);
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static LiteralOp replaceLiteralValueTypeCastRightIndexing( Hop c, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- LiteralOp ret = null;
-
- //as.scalar/right indexing w/ literals/vars and matrix less than 10^6 cells
- if( c instanceof UnaryOp && ((UnaryOp)c).getOp() == OpOp1.CAST_AS_SCALAR
- && c.getInput().get(0) instanceof IndexingOp )
- {
- IndexingOp rix = (IndexingOp)c.getInput().get(0);
- Hop data = rix.getInput().get(0);
- Hop rl = rix.getInput().get(1);
- Hop ru = rix.getInput().get(2);
- Hop cl = rix.getInput().get(3);
- Hop cu = rix.getInput().get(4);
- if( rix.dimsKnown() && rix.getDim1()==1 && rix.getDim2()==1
- && data instanceof DataOp && vars.keySet().contains(data.getName())
- && isIntValueDataLiteral(rl, vars) && isIntValueDataLiteral(ru, vars)
- && isIntValueDataLiteral(cl, vars) && isIntValueDataLiteral(cu, vars) )
- {
- long rlval = getIntValueDataLiteral(rl, vars);
- long clval = getIntValueDataLiteral(cl, vars);
-
- MatrixObject mo = (MatrixObject)vars.get(data.getName());
-
- //get the dimension information from the matrix object because the hop
- //dimensions might not have been updated during recompile
- if( mo.getNumRows()*mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE )
- {
- MatrixBlock mBlock = mo.acquireRead();
- double value = mBlock.getValue((int)rlval-1,(int)clval-1);
- mo.release();
-
- //literal substitution (always double)
- ret = new LiteralOp(value);
- }
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static LiteralOp replaceLiteralFullUnaryAggregate( Hop c, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- LiteralOp ret = null;
-
- //full unary aggregate w/ matrix less than 10^6 cells
- if( c instanceof AggUnaryOp
- && isReplaceableUnaryAggregate((AggUnaryOp)c)
- && c.getInput().get(0) instanceof DataOp
- && vars.keySet().contains(c.getInput().get(0).getName()) )
- {
- Hop data = c.getInput().get(0);
- MatrixObject mo = (MatrixObject) vars.get(data.getName());
-
- //get the dimension information from the matrix object because the hop
- //dimensions might not have been updated during recompile
- if( mo.getNumRows()*mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE )
- {
- MatrixBlock mBlock = mo.acquireRead();
- double value = replaceUnaryAggregate((AggUnaryOp)c, mBlock);
- mo.release();
-
- //literal substitution (always double)
- ret = new LiteralOp(value);
- }
- }
-
- return ret;
- }
-
- /**
- *
- * @param c
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static LiteralOp replaceLiteralFullUnaryAggregateRightIndexing( Hop c, LocalVariableMap vars )
- throws DMLRuntimeException
- {
- LiteralOp ret = null;
-
- //full unary aggregate w/ indexed matrix less than 10^6 cells
- if( c instanceof AggUnaryOp
- && isReplaceableUnaryAggregate((AggUnaryOp)c)
- && c.getInput().get(0) instanceof IndexingOp
- && c.getInput().get(0).getInput().get(0) instanceof DataOp )
- {
- IndexingOp rix = (IndexingOp)c.getInput().get(0);
- Hop data = rix.getInput().get(0);
- Hop rl = rix.getInput().get(1);
- Hop ru = rix.getInput().get(2);
- Hop cl = rix.getInput().get(3);
- Hop cu = rix.getInput().get(4);
-
- if( data instanceof DataOp && vars.keySet().contains(data.getName())
- && isIntValueDataLiteral(rl, vars) && isIntValueDataLiteral(ru, vars)
- && isIntValueDataLiteral(cl, vars) && isIntValueDataLiteral(cu, vars) )
- {
- long rlval = getIntValueDataLiteral(rl, vars);
- long ruval = getIntValueDataLiteral(ru, vars);
- long clval = getIntValueDataLiteral(cl, vars);
- long cuval = getIntValueDataLiteral(cu, vars);
-
- MatrixObject mo = (MatrixObject) vars.get(data.getName());
-
- //get the dimension information from the matrix object because the hop
- //dimensions might not have been updated during recompile
- if( mo.getNumRows()*mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE )
- {
- MatrixBlock mBlock = mo.acquireRead();
- MatrixBlock mBlock2 = mBlock.sliceOperations((int)(rlval-1), (int)(ruval-1), (int)(clval-1), (int)(cuval-1), new MatrixBlock());
- double value = replaceUnaryAggregate((AggUnaryOp)c, mBlock2);
- mo.release();
-
- //literal substitution (always double)
- ret = new LiteralOp(value);
- }
- }
- }
-
- return ret;
- }
-
-
- ///////////////////////////////
- // Utility functions
- ///////////////////////////////
-
- /**
- *
- * @param h
- * @param vars
- * @return
- */
- private static boolean isIntValueDataLiteral(Hop h, LocalVariableMap vars)
- {
- return ( (h instanceof DataOp && vars.keySet().contains(h.getName()))
- || h instanceof LiteralOp
- ||(h instanceof UnaryOp && (((UnaryOp)h).getOp()==OpOp1.NROW || ((UnaryOp)h).getOp()==OpOp1.NCOL)
- && h.getInput().get(0) instanceof DataOp && vars.keySet().contains(h.getInput().get(0).getName())) );
- }
-
- /**
- *
- * @param hop
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
- private static long getIntValueDataLiteral(Hop hop, LocalVariableMap vars)
- throws DMLRuntimeException
- {
- long value = -1;
-
- try
- {
- if( hop instanceof LiteralOp )
- {
- value = HopRewriteUtils.getIntValue((LiteralOp)hop);
- }
- else if( hop instanceof UnaryOp && ((UnaryOp)hop).getOp()==OpOp1.NROW )
- {
- //get the dimension information from the matrix object because the hop
- //dimensions might not have been updated during recompile
- MatrixObject mo = (MatrixObject)vars.get(hop.getInput().get(0).getName());
- value = mo.getNumRows();
- }
- else if( hop instanceof UnaryOp && ((UnaryOp)hop).getOp()==OpOp1.NCOL )
- {
- //get the dimension information from the matrix object because the hop
- //dimensions might not have been updated during recompile
- MatrixObject mo = (MatrixObject)vars.get(hop.getInput().get(0).getName());
- value = mo.getNumColumns();
- }
- else
- {
- ScalarObject sdat = (ScalarObject) vars.get(hop.getName());
- value = sdat.getLongValue();
- }
- }
- catch(HopsException ex)
- {
- throw new DMLRuntimeException("Failed to get int value for literal replacement", ex);
- }
-
- return value;
- }
-
-
- /**
- *
- * @param auop
- * @return
- */
- private static boolean isReplaceableUnaryAggregate( AggUnaryOp auop )
- {
- boolean cdir = (auop.getDirection() == Direction.RowCol);
- boolean cop = ( auop.getOp() == AggOp.SUM
- || auop.getOp() == AggOp.SUM_SQ
- || auop.getOp() == AggOp.MIN
- || auop.getOp() == AggOp.MAX );
-
- return cdir && cop;
- }
-
- /**
- *
- * @param auop
- * @param mb
- * @return
- * @throws DMLRuntimeException
- */
- private static double replaceUnaryAggregate( AggUnaryOp auop, MatrixBlock mb )
- throws DMLRuntimeException
- {
- //setup stats reporting if necessary
- boolean REPORT_STATS = (DMLScript.STATISTICS && REPORT_LITERAL_REPLACE_OPS_STATS);
- long t0 = REPORT_STATS ? System.nanoTime() : 0;
-
- //compute required unary aggregate
- double val = Double.MAX_VALUE;
- switch( auop.getOp() ) {
- case SUM:
- val = mb.sum();
- break;
- case SUM_SQ:
- val = mb.sumSq();
- break;
- case MIN:
- val = mb.min();
- break;
- case MAX:
- val = mb.max();
- break;
- default:
- throw new DMLRuntimeException("Unsupported unary aggregate replacement: "+auop.getOp());
- }
-
- //report statistics if necessary
- if( REPORT_STATS ){
- long t1 = System.nanoTime();
- Statistics.maintainCPHeavyHitters("rlit", t1-t0);
- }
-
- return val;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/recompile/RecompileStatus.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/recompile/RecompileStatus.java b/src/main/java/com/ibm/bi/dml/hops/recompile/RecompileStatus.java
deleted file mode 100644
index e6b8426..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/recompile/RecompileStatus.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.recompile;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-public class RecompileStatus
-{
-
- private HashMap<String, MatrixCharacteristics> _lastTWrites = null;
-
- public RecompileStatus()
- {
- _lastTWrites = new HashMap<String,MatrixCharacteristics>();
- }
-
- public HashMap<String, MatrixCharacteristics> getTWriteStats()
- {
- return _lastTWrites;
- }
-
- public void clearStatus()
- {
- _lastTWrites.clear();
- }
-
- public void clearStatus(VariableSet vars)
- {
- ArrayList<String> lvars = new ArrayList<String>(vars.getVariableNames());
- for( String var : lvars ) {
- _lastTWrites.remove(var);
- }
- }
-
- @Override
- public Object clone()
- {
- RecompileStatus ret = new RecompileStatus();
- ret._lastTWrites.putAll(this._lastTWrites);
- return ret;
- }
-}
[17/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ParForStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParForStatement.java b/src/main/java/com/ibm/bi/dml/parser/ParForStatement.java
deleted file mode 100644
index c8b1bf6..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ParForStatement.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-/**
- * This ParForStatement is essentially identical to a ForStatement, except an extended
- * toString method for printing the 'parfor' keyword.
- *
- *
- */
-public class ParForStatement extends ForStatement
-{
-
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("parfor ");
- sb.append(_predicate.toString());
- sb.append(" { \n");
- for (StatementBlock block : _body){
- sb.append(block.toString());
- }
- sb.append("}\n");
- return sb.toString();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ParForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParForStatementBlock.java b/src/main/java/com/ibm/bi/dml/parser/ParForStatementBlock.java
deleted file mode 100644
index e2ae2d7..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ParForStatementBlock.java
+++ /dev/null
@@ -1,2177 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.parser.Expression.BinaryOp;
-import com.ibm.bi.dml.parser.Expression.BuiltinFunctionOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PExecMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.POptMode;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PTaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
-
-/**
- *
- * This ParForStatementBlock is essentially identical to a ForStatementBlock, except an extended validate
- * for checking/setting optional parfor parameters and running the loop dependency analysis.
- *
- * TODO range bounds which depend on iteration variable (currently too conservative)
- *
- */
-public class ParForStatementBlock extends ForStatementBlock
-{
-
- private static final boolean LDEBUG = false; //internal local debug level
- private static final Log LOG = LogFactory.getLog(ParForStatementBlock.class.getName());
-
- //external parameter names
- private static HashSet<String> _paramNames;
- public static final String CHECK = "check"; //run loop dependency analysis
- public static final String PAR = "par"; //number of parallel workers
- public static final String TASK_PARTITIONER = "taskpartitioner"; //task partitioner
- public static final String TASK_SIZE = "tasksize"; //number of tasks
- public static final String DATA_PARTITIONER = "datapartitioner"; //task partitioner
- public static final String RESULT_MERGE = "resultmerge"; //task partitioner
- public static final String EXEC_MODE = "mode"; //runtime execution mode
- public static final String OPT_MODE = "opt"; //runtime execution mode
- public static final String OPT_LOG = "log"; //parfor logging mode
- public static final String PROFILE = "profile"; //monitor and report parfor performance profile
-
- //default external parameter values
- private static HashMap<String, String> _paramDefaults;
- private static HashMap<String, String> _paramDefaults2; //for constrained opt
-
- //internal parameter values
- private static final boolean NORMALIZE = false; //normalize FOR from to incr
- private static final boolean USE_FN_CACHE = false; //useful for larger scripts (due to O(n^2))
- private static final boolean ABORT_ON_FIRST_DEPENDENCY = true;
- private static final boolean CONSERVATIVE_CHECK = false; //include FOR into dep analysis, reject unknown vars (otherwise use internal vars for whole row or column)
-
-
- public static final String INTERAL_FN_INDEX_ROW = "__ixr"; //pseudo index for range indexing row
- public static final String INTERAL_FN_INDEX_COL = "__ixc"; //pseudo index for range indexing col
-
- //class members
- private static IDSequence _idSeq = null;
- private static IDSequence _idSeqfn = null;
-
- private static HashMap<String, LinearFunction> _fncache; //slower for most (small cases) cases
-
- //instance members
- private long _ID = -1;
- private VariableSet _vsParent = null;
- private ArrayList<String> _resultVars = null;
- private Bounds _bounds = null;
-
- static
- {
- // populate parameter name lookup-table
- _paramNames = new HashSet<String>();
- _paramNames.add( CHECK );
- _paramNames.add( PAR );
- _paramNames.add( TASK_PARTITIONER );
- _paramNames.add( TASK_SIZE );
- _paramNames.add( DATA_PARTITIONER );
- _paramNames.add( RESULT_MERGE );
- _paramNames.add( EXEC_MODE );
- _paramNames.add( OPT_MODE );
- _paramNames.add( PROFILE );
- _paramNames.add( OPT_LOG );
-
- // populate defaults lookup-table
- _paramDefaults = new HashMap<String, String>();
- _paramDefaults.put( CHECK, "1" );
- _paramDefaults.put( PAR, String.valueOf(InfrastructureAnalyzer.getLocalParallelism()) );
- _paramDefaults.put( TASK_PARTITIONER, String.valueOf(PTaskPartitioner.FIXED) );
- _paramDefaults.put( TASK_SIZE, "1" );
- _paramDefaults.put( DATA_PARTITIONER, String.valueOf(PDataPartitioner.NONE) );
- _paramDefaults.put( RESULT_MERGE, String.valueOf(PResultMerge.LOCAL_AUTOMATIC) );
- _paramDefaults.put( EXEC_MODE, String.valueOf(PExecMode.LOCAL) );
- _paramDefaults.put( OPT_MODE, String.valueOf(POptMode.RULEBASED) );
- _paramDefaults.put( PROFILE, "0" );
- _paramDefaults.put( OPT_LOG, Logger.getRootLogger().getLevel().toString() );
-
- _paramDefaults2 = new HashMap<String, String>(); //OPT_MODE always specified
- _paramDefaults2.put( CHECK, "1" );
- _paramDefaults2.put( PAR, "-1" );
- _paramDefaults2.put( TASK_PARTITIONER, String.valueOf(PTaskPartitioner.UNSPECIFIED) );
- _paramDefaults2.put( TASK_SIZE, "-1" );
- _paramDefaults2.put( DATA_PARTITIONER, String.valueOf(PDataPartitioner.UNSPECIFIED) );
- _paramDefaults2.put( RESULT_MERGE, String.valueOf(PResultMerge.UNSPECIFIED) );
- _paramDefaults2.put( EXEC_MODE, String.valueOf(PExecMode.UNSPECIFIED) );
- _paramDefaults2.put( PROFILE, "0" );
- _paramDefaults2.put( OPT_LOG, Logger.getRootLogger().getLevel().toString() );
-
- _idSeq = new IDSequence();
- _idSeqfn = new IDSequence();
-
- //initialize function cache
- if( USE_FN_CACHE ) {
- _fncache = new HashMap<String, LinearFunction>();
- }
-
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.parser.ParForStatementBlock")
- .setLevel((Level) Level.TRACE);
- }
- }
-
- public ParForStatementBlock()
- {
- _ID = _idSeq.getNextID();
- _resultVars = new ArrayList<String>();
-
- LOG.trace("PARFOR("+_ID+"): ParForStatementBlock instance created");
- }
-
- public long getID()
- {
- return _ID;
- }
-
- public ArrayList<String> getResultVariables()
- {
- return _resultVars;
- }
-
- private void addToResultVariablesNoDup( String var )
- {
- if( !_resultVars.contains( var ) )
- _resultVars.add( var );
- }
-
- @Override
- public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String,ConstIdentifier> constVars, boolean conditional)
- throws LanguageException, ParseException, IOException
- {
- LOG.trace("PARFOR("+_ID+"): validating ParForStatementBlock.");
-
- //create parent variable set via cloning
- _vsParent = new VariableSet( ids );
-
- if(LOG.isTraceEnabled()) //note: A is matrix, and A[i,1] is scalar
- for( DataIdentifier di : _vsParent.getVariables().values() )
- LOG.trace("PARFOR: non-local "+di._name+": "+di.getDataType().toString()+" with rowDim = "+di.getDim1());
-
- //normal validate via ForStatement (sequential)
- //NOTES:
- // * validate/dependency checking of nested parfor-loops happens at this point
- // * validate includes also constant propagation for from, to, incr expressions
- // * this includes also function inlining
- VariableSet vs = super.validate(dmlProg, ids, constVars, conditional);
-
- //check of correctness of specified parfor parameter names and
- //set default parameter values for all not specified parameters
- ParForStatement pfs = (ParForStatement) _statements.get(0);
- IterablePredicate predicate = pfs.getIterablePredicate();
- HashMap<String, String> params = predicate.getParForParams();
- if( params != null ) //if parameter specified
- {
- //check for valid parameter types
- for( String key : params.keySet() )
- if( !_paramNames.contains(key) ){ //always unconditional
- raiseValidateError("PARFOR: The specified parameter '"+key+"' is no valid parfor parameter.", false);
- }
- //set defaults for all non-specified values
- //(except if CONSTRAINT optimizer, in order to distinguish specified parameters)
- boolean constrained = (params.containsKey( OPT_MODE ) && params.get( OPT_MODE ).equals(POptMode.CONSTRAINED.toString()));
- for( String key : _paramNames )
- if( !params.containsKey(key) )
- {
- if( constrained )
- {
- params.put(key, _paramDefaults2.get(key));
- }
- //special treatment for degree of parallelism
- else if( key.equals(PAR) && params.containsKey(EXEC_MODE)
- && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR.toString()))
- {
- int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() )
- maxPMap = (int)Math.max( maxPMap, YarnClusterAnalyzer.getNumCores() );
- params.put(key, String.valueOf(maxPMap));
- }
- else if( key.equals(PAR) && params.containsKey(EXEC_MODE)
- && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR_DP.toString()) )
- {
- int maxPRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() )
- maxPRed = (int)Math.max( maxPRed, YarnClusterAnalyzer.getNumCores()/2 );
- params.put(key, String.valueOf(maxPRed));
- }
- else //default case
- params.put(key, _paramDefaults.get(key));
- }
-
- //check for disabled parameters values
- if( params.containsKey( OPT_MODE ) )
- {
- String optStr = params.get( OPT_MODE );
- if( optStr.equals(POptMode.HEURISTIC.toString())
- || optStr.equals(POptMode.GREEDY.toString())
- || optStr.equals(POptMode.FULL_DP.toString()) )
- { //always unconditional
- raiseValidateError("Sorry, parfor optimization mode '"+optStr+"' is disabled for external usage.", false);
- }
- }
-
- }
- else
- {
- //set all defaults
- params = new HashMap<String, String>();
- params.putAll( _paramDefaults );
- predicate.setParForParams(params);
- }
-
- //start time measurement for normalization and dependency analysis
- Timing time = new Timing(true);
-
- // LOOP DEPENDENCY ANALYSIS (test for dependency existence)
- // no false negative guaranteed, but possibly false positives
-
- /* Basic intuition: WRITES to NON-local variables are only permitted iff
- * - no data dep (no read other than own iteration w i < r j)
- * - no anti dep (no read other than own iteration w i > r j)
- * - no output dep (no write other than own iteration)
- *
- * ALGORITHM:
- * 1) Determine candidates C (writes to non-local variables)
- * 2) Prune all c from C where no dependencies --> C'
- * 3) Raise an exception/warning if C' not the empty set
- *
- * RESTRICTIONS:
- * - array subscripts of non-local variables must be linear functions of the form
- * a0+ a1*i + ... + a2*j, where i and j are for or parfor indexes.
- * - for and parfor increments must be integer values
- * - only static (integer lower, upper bounds) range indexing
- * - only input variables considered as potential candidates for checking
- *
- * (TODO: in order to remove the last restriction, dependencies must be checked again after
- * live variable analysis against LIVEOUT)
- *
- * NOTE: validity is only checked during compilation, i.e., for dynamic from, to, incr MIN MAX values assumed.
- */
-
- LOG.trace("PARFOR: running loop dependency analysis ...");
-
- //### Step 1 ###: determine candidate set C
- HashSet<Candidate> C = new HashSet<Candidate>();
- HashSet<Candidate> C2 = new HashSet<Candidate>();
- Integer sCount = 0; //object for call by ref
- rDetermineCandidates(pfs.getBody(), C, sCount);
-
- boolean check = (Integer.parseInt(params.get(CHECK))==1);
- if( check )
- {
- //### Step 2 ###: prune c without dependencies
- _bounds = new Bounds();
- for( FunctionStatementBlock fsb : dmlProg.getFunctionStatementBlocks() )
- rDetermineBounds( fsb, false ); //writes to _bounds
- rDetermineBounds( dmlProg.getStatementBlocks(), false ); //writes to _bounds
-
- for( Candidate c : C )
- {
- DataType cdt = _vsParent.getVariables().get(c._var).getDataType(); //might be different in DataIdentifier
-
- //assume no dependency
- sCount = 0;
- boolean[] dep = new boolean[]{false,false,false}; //output, data, anti
- rCheckCandidates(c, cdt, pfs.getBody(), sCount, dep);
-
-
- if (LOG.isTraceEnabled())
- {
- if( dep[0] )
- LOG.trace("PARFOR: output dependency detected for var '"+c._var+"'.");
- if( dep[1] )
- LOG.trace("PARFOR: data dependency detected for var '"+c._var+"'.");
- if( dep[2] )
- LOG.trace("PARFOR: anti dependency detected for var '"+c._var+"'.");
- }
-
- if( dep[0] || dep[1] || dep[2] )
- {
- C2.add(c);
- if( ABORT_ON_FIRST_DEPENDENCY )
- break;
- }
- }
-
-
- //### Step 3 ###: raise an exception / warning
- if( C2.size() > 0 )
- {
- LOG.trace("PARFOR: loop dependencies detected.");
-
- StringBuilder depVars = new StringBuilder();
- for( Candidate c : C2 )
- {
- if( depVars.length()>0 )
- depVars.append(", ");
- depVars.append(c._var);
- }
-
- //always unconditional (to ensure we always raise dependency issues)
- raiseValidateError("PARFOR loop dependency analysis: " +
- "inter-iteration (loop-carried) dependencies detected for variable(s): " +
- depVars.toString() +". \n " +
- "Please, ensure independence of iterations.", false);
- }
- else
- {
- LOG.trace("PARFOR: no loop dependencies detected.");
- }
-
- }
- else
- {
- LOG.debug("INFO: PARFOR("+_ID+"): loop dependency analysis skipped.");
- }
-
- //if successful, prepare result variables (all distinct vars in all candidates)
- //a) add own candidates
- for( Candidate var : C )
- if( check || var._dat.getDataType()!=DataType.SCALAR )
- addToResultVariablesNoDup( var._var );
- //b) get and add child result vars (if required)
- ArrayList<String> tmp = new ArrayList<String>();
- rConsolidateResultVars(pfs.getBody(), tmp);
- for( String var : tmp )
- if(_vsParent.containsVariable(var))
- addToResultVariablesNoDup( var );
- if( LDEBUG )
- for( String rvar : _resultVars )
- LOG.debug("INFO: PARFOR final result variable: "+rvar);
-
- //cleanup function cache in order to prevent side effects between parfor statements
- if( USE_FN_CACHE )
- _fncache.clear();
-
- LOG.debug("INFO: PARFOR("+_ID+"): validate successful (no dependencies) in "+time.stop()+"ms.");
-
- return vs;
- }
-
- /**
- *
- * @param sb
- * @return
- */
- public ArrayList<String> getReadOnlyParentVars()
- {
- ArrayList<String> ret = new ArrayList<String>();
-
- VariableSet read = variablesRead();
- VariableSet updated = variablesUpdated();
- VariableSet livein = liveIn();
- for( String var : livein.getVariableNames() ) //for all parent variables
- if( read.containsVariable(var) && !updated.containsVariable(var) ) //read-only
- ret.add( var );
-
- return ret;
- }
-
- /**
- * Determines the PDataPartitioningFormat for read-only parent variables according
- * to the access pattern of that variable within the parfor statement block.
- * Row-wise or column wise partitioning is only suggested if we see pure row-wise or
- * column-wise access patterns.
- *
- * @param var
- * @return
- */
- public PDataPartitionFormat determineDataPartitionFormat(String var)
- {
- PDataPartitionFormat dpf = null;
- List<PDataPartitionFormat> dpfc = new LinkedList<PDataPartitionFormat>();
-
- try
- {
- //determine partitioning candidates
- ParForStatement pfs = (ParForStatement) _statements.get(0);
- rDeterminePartitioningCandidates(var, pfs.getBody(), dpfc);
-
- //determine final solution
- for( PDataPartitionFormat tmp : dpfc )
- {
- //System.out.println(var+": "+tmp);
- if( dpf != null && dpf!=tmp ) //if no consensus
- dpf = PDataPartitionFormat.NONE;
- else
- dpf = tmp;
-
- /* TODO block partitioning
- if( dpf == null || dpf==tmp ) //consensus
- dpf = tmp;
- else if( dpf==PDataPartitionFormat.BLOCK_WISE_M_N //subsumption
- || tmp==PDataPartitionFormat.BLOCK_WISE_M_N )
- dpf = PDataPartitionFormat.BLOCK_WISE_M_N;
- else //no consensus
- dpf = PDataPartitionFormat.NONE;
- */
- }
- if( dpf == null )
- dpf = PDataPartitionFormat.NONE;
- }
- catch (LanguageException e)
- {
- LOG.trace( "Unable to determine partitioning candidates.", e );
- dpf = PDataPartitionFormat.NONE;
- }
-
- return dpf;
- }
-
- /**
- * This method recursively determines candidates for output,data,anti dependencies.
- * Candidates are defined as writes to non-local variables.
- *
- * @param asb
- * @param C
- * @param sCount
- * @throws LanguageException
- */
- private void rDetermineCandidates(ArrayList<StatementBlock> asb, HashSet<Candidate> C, Integer sCount)
- throws LanguageException
- {
- for(StatementBlock sb : asb ) // foreach statementblock in parforbody
- for( Statement s : sb._statements ) // foreach statement in statement block
- {
- sCount++;
-
- if( s instanceof ForStatement ) //incl parfor
- {
- //despite separate dependency analysis for each nested parfor, we need to
- //recursively check nested parfor as well in order to ensure correcteness
- //of constantChecks with regard to outer indexes
- rDetermineCandidates(((ForStatement)s).getBody(), C, sCount);
- }
- else if( s instanceof WhileStatement )
- {
- rDetermineCandidates(((WhileStatement)s).getBody(), C, sCount);
- }
- else if( s instanceof IfStatement )
- {
- rDetermineCandidates(((IfStatement)s).getIfBody(), C, sCount);
- rDetermineCandidates(((IfStatement)s).getElseBody(), C, sCount);
- }
- else if( s instanceof FunctionStatement )
- {
- rDetermineCandidates(((FunctionStatement)s).getBody(), C, sCount);
- }
- else if( s instanceof PrintStatement && ((PrintStatement)s).getType() == PRINTTYPE.STOP ) {
- raiseValidateError("PARFOR loop dependency analysis: " +
- "stop() statement is not allowed inside a parfor loop body." , false);
- }
- else
- {
- VariableSet vsUpdated = s.variablesUpdated();
- if( vsUpdated != null )
- for(String write : vsUpdated.getVariableNames())
- {
- //add writes to non-local variables to candidate set
- if( _vsParent.containsVariable(write) )
- {
- List<DataIdentifier> dats = getDataIdentifiers( s, true );
- for( DataIdentifier dat : dats )
- {
- Candidate c = new Candidate();
- c._var = write;
- c._dat = dat;
- C.add( c );
- }
-
- LOG.trace("PARFOR: dependency candidate: var '"+write+"'");
- }
- }
- }
- }
- }
-
- /**
- * This method recursively determines partitioning candidates for input variables.
- * Candidates are defined as index reads of non-local variables.
- *
- * @param asb
- * @param C
- * @throws LanguageException
- */
- private void rDeterminePartitioningCandidates(String var, ArrayList<StatementBlock> asb, List<PDataPartitionFormat> C)
- throws LanguageException
- {
- for(StatementBlock sb : asb ) // foreach statementblock in parforbody
- for( Statement s : sb._statements ) // foreach statement in statement block
- {
- if( s instanceof ForStatement ) //includes for and parfor
- {
- ForStatement fs = (ForStatement) s;
- //predicate
- List<DataIdentifier> datsFromRead = rGetDataIdentifiers(fs.getIterablePredicate().getFromExpr());
- List<DataIdentifier> datsToRead = rGetDataIdentifiers(fs.getIterablePredicate().getToExpr());
- List<DataIdentifier> datsIncrementRead = rGetDataIdentifiers(fs.getIterablePredicate().getIncrementExpr());
- rDeterminePartitioningCandidates(var, datsFromRead, C);
- rDeterminePartitioningCandidates(var, datsToRead, C);
- rDeterminePartitioningCandidates(var, datsIncrementRead, C);
- //for / parfor body
- rDeterminePartitioningCandidates(var,((ForStatement)s).getBody(), C);
- }
- else if( s instanceof WhileStatement )
- {
- WhileStatement ws = (WhileStatement) s;
- //predicate
- List<DataIdentifier> datsRead = rGetDataIdentifiers(ws.getConditionalPredicate().getPredicate());
- rDeterminePartitioningCandidates(var, datsRead, C);
- //while body
- rDeterminePartitioningCandidates(var,((WhileStatement)s).getBody(), C);
- }
- else if( s instanceof IfStatement )
- {
- IfStatement is = (IfStatement) s;
- //predicate
- List<DataIdentifier> datsRead = rGetDataIdentifiers(is.getConditionalPredicate().getPredicate());
- rDeterminePartitioningCandidates(var, datsRead, C);
- //if and else branch
- rDeterminePartitioningCandidates(var,((IfStatement)s).getIfBody(), C);
- rDeterminePartitioningCandidates(var,((IfStatement)s).getElseBody(), C);
- }
- else if( s instanceof FunctionStatement )
- {
- rDeterminePartitioningCandidates(var,((FunctionStatement)s).getBody(), C);
- }
- else
- {
- List<DataIdentifier> datsRead = getDataIdentifiers(s, false);
- rDeterminePartitioningCandidates(var, datsRead, C);
- }
- }
- }
-
- /**
- *
- * @param var
- * @param datsRead
- * @param C
- */
- private void rDeterminePartitioningCandidates(String var, List<DataIdentifier> datsRead, List<PDataPartitionFormat> C)
- {
- if( datsRead != null )
- for(DataIdentifier read : datsRead)
- {
- String readStr = read.getName();
- if( var.equals( readStr ) )
- {
- if( read instanceof IndexedIdentifier )
- {
- IndexedIdentifier idat = (IndexedIdentifier) read;
- C.add( determineAccessPattern(idat) );
- }
- else if( read instanceof DataIdentifier )
- {
- C.add( PDataPartitionFormat.NONE );
- }
- }
- }
- }
-
- private PDataPartitionFormat determineAccessPattern( IndexedIdentifier dat )
- {
- PDataPartitionFormat dpf = null;
-
- //1) get all bounds expressions for index access
- Expression rowL = dat.getRowLowerBound();
- Expression rowU = dat.getRowUpperBound();
- Expression colL = dat.getColLowerBound();
- Expression colU = dat.getColUpperBound();
-
- //2) decided on access pattern
- //COLUMN_WISE iff row expr is colon (all rows)
- // and access to single column
- if( rowL == null && rowU == null &&
- colL!=null && colU != null && colL.equals(colU) )
- {
- dpf = PDataPartitionFormat.COLUMN_WISE;
- }
- //ROW_WISE iff col expr is colon (all columns)
- // and access to single row
- else if( colL == null && colU == null &&
- rowL!=null && rowU != null && rowL.equals(rowU) )
- {
- dpf = PDataPartitionFormat.ROW_WISE;
- }
- //NONE otherwise (conservative)
- else
- dpf = PDataPartitionFormat.NONE;
-
- //TODO block partitioning
-
- return dpf;
- }
-
- private void rConsolidateResultVars(ArrayList<StatementBlock> asb, ArrayList<String> vars)
- throws LanguageException
- {
- for(StatementBlock sb : asb ) // foreach statementblock in parforbody
- {
- if( sb instanceof ParForStatementBlock )
- {
- vars.addAll(((ParForStatementBlock)sb).getResultVariables());
- }
-
- for( Statement s : sb._statements ) // foreach statement in statement block
- {
- if( s instanceof ForStatement || s instanceof ParForStatement )
- {
- rConsolidateResultVars(((ForStatement)s).getBody(), vars);
- }
- else if( s instanceof WhileStatement )
- {
- rConsolidateResultVars(((WhileStatement)s).getBody(), vars);
- }
- else if( s instanceof IfStatement )
- {
- rConsolidateResultVars(((IfStatement)s).getIfBody(), vars);
- rConsolidateResultVars(((IfStatement)s).getElseBody(), vars);
- }
- else if( s instanceof FunctionStatement )
- {
- rConsolidateResultVars(((FunctionStatement)s).getBody(), vars);
- }
- }
- }
- }
-
- /**
- * This method recursively checks a candidate against StatementBlocks for anti, data and output dependencies.
- * A LanguageException is raised if at least one dependency is found, where it is guaranteed that no false negatives
- * (undetected dependency) but potentially false positives (misdetected dependency) can appear.
- *
- *
- * @param c
- * @param cdt
- * @param asb
- * @param sCount
- * @param dep
- * @throws LanguageException
- */
- private void rCheckCandidates(Candidate c, DataType cdt, ArrayList<StatementBlock> asb,
- Integer sCount, boolean[] dep)
- throws LanguageException
- {
- // check candidate only (output dependency if scalar or constant matrix subscript)
- if( cdt == DataType.SCALAR
- || cdt == DataType.OBJECT ) //dat2 checked for other candidate
- {
- //every write to a scalar or complete data object is an output dependency
- dep[0] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
- }
- else if( cdt == DataType.MATRIX )
- {
- if( runConstantCheck(c._dat) )
- {
- LOG.trace("PARFOR: Possible output dependency detected via constant self-check: var '"+c._var+"'.");
- dep[0] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
- }
- }
-
- // check candidate against all statements
- for(StatementBlock sb : asb )
- for( Statement s : sb._statements )
- {
- sCount++;
-
- if( s instanceof ForStatement ) //incl parfor
- {
- //despite separate dependency analysis for each nested parfor, we need to
- //recursively check nested parfor as well in order to ensure correcteness
- //of constantChecks with regard to outer indexes
- rCheckCandidates(c, cdt, ((ForStatement)s).getBody(), sCount, dep);
- }
- else if( s instanceof WhileStatement )
- {
- rCheckCandidates(c, cdt, ((WhileStatement)s).getBody(), sCount, dep);
- }
- else if( s instanceof IfStatement )
- {
- rCheckCandidates(c, cdt, ((IfStatement)s).getIfBody(), sCount, dep);
- rCheckCandidates(c, cdt, ((IfStatement)s).getElseBody(), sCount, dep);
- }
- else if( s instanceof FunctionStatement )
- {
- rCheckCandidates(c, cdt, ((FunctionStatement)s).getBody(), sCount, dep);
- }
- else
- {
- //CHECK output dependencies
- List<DataIdentifier> datsUpdated = getDataIdentifiers(s, true);
-
- if( datsUpdated != null )
- for(DataIdentifier write : datsUpdated)
- {
- String writeStr = write.getName();
- if( c._var.equals( writeStr ) )
- {
- DataIdentifier dat2 = write;
-
- if( cdt == DataType.MATRIX )
- {
- if( c._dat != dat2 ) //omit self-check
- {
- if( runEqualsCheck(c._dat, dat2) )
- {
- //intra-iteration output dependencies (same index function) are OK
- }
- else if(runBanerjeeGCDTest( c._dat, dat2 ))
- {
- LOG.trace("PARFOR: Possible output dependency detected via GCD/Banerjee: var '"+write+"'.");
- dep[0] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
- }
- }
- }
- else // at least one type UNKNOWN
- {
- //cannot infer type, need to exit (conservative approach)
- throw new LanguageException("PARFOR loop dependency analysis: cannot check for dependencies " +
- "due to unknown datatype of var '"+c._var+"'.");
- }
- }
- }
-
- List<DataIdentifier> datsRead = getDataIdentifiers(s, false);
-
- //check data and anti dependencies
- if( datsRead != null )
- for(DataIdentifier read : datsRead)
- {
- String readStr = read.getName();
-
- if( c._var.equals( readStr ) )
- {
- DataIdentifier dat2 = read;
- DataType dat2dt = _vsParent.getVariables().get(readStr).getDataType(); //vs.getVariables().get(read).getDataType();
-
- if( cdt == DataType.SCALAR
- || cdt == DataType.OBJECT
- || dat2dt == DataType.SCALAR
- || dat2dt == DataType.OBJECT )
- {
- //every write, read combination involving a scalar is a data dependency
- dep[1] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
-
- //if(!output) //no write before read in iteration body
- //data = true;
-
- }
- else if( cdt == DataType.MATRIX
- && dat2dt == DataType.MATRIX )
- {
- if( runEqualsCheck(c._dat, dat2) )
- {
- //read after write on same index, and not constant (checked for output)
- //is OK
- }
- else if( runBanerjeeGCDTest( c._dat, dat2 ) )
- {
- LOG.trace("PARFOR: Possible data/anti dependency detected via GCD/Banerjee: var '"+read+"'.");
- dep[1] = true;
- dep[2] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
- }
- else if( !(dat2 instanceof IndexedIdentifier) )
- {
- //non-indexed access to candidate result variable -> always a dependency
- LOG.trace("PARFOR: Possible data/anti dependency detected via GCD/Banerjee: var '"+read+"'.");
- dep[1] = true;
- dep[2] = true;
- if( ABORT_ON_FIRST_DEPENDENCY )
- return;
- }
- }
- else //if( c._dat.getDataType() == DataType.UNKNOWN )
- {
- //cannot infer type, need to exit (conservative approach)
- throw new LanguageException("PARFOR loop dependency analysis: cannot check for dependencies " +
- "due to unknown datatype of var '"+c._var+"'.");
- }
- }
- }
- }
- }
- }
-
- /**
- * Get all target/source DataIdentifiers of the given statement.
- *
- * @param s
- * @param target
- * @return
- */
- private List<DataIdentifier> getDataIdentifiers(Statement s, boolean target)
- {
- List<DataIdentifier> ret = null;
-
- if( s instanceof AssignmentStatement )
- {
- AssignmentStatement s2 = (AssignmentStatement)s;
- if(target)
- ret = s2.getTargetList();
- else
- ret = rGetDataIdentifiers(s2.getSource());
- }
- else if (s instanceof FunctionStatement)
- {
- FunctionStatement s2 = (FunctionStatement)s;
- if(target)
- ret = s2.getOutputParams();
- else
- ret = s2.getInputParams();
- }
- else if (s instanceof MultiAssignmentStatement)
- {
- MultiAssignmentStatement s2 = (MultiAssignmentStatement)s;
- if(target)
- ret = s2.getTargetList();
- else
- ret = rGetDataIdentifiers(s2.getSource());
- }
- else if (s instanceof PrintStatement)
- {
- PrintStatement s2 = (PrintStatement)s;
- ret = rGetDataIdentifiers(s2.getExpression());
- }
-
- //potentially extend this list with other Statements if required
- //(e.g., IOStatement, RandStatement)
-
- return ret;
- }
-
- private boolean isRowIgnorable(IndexedIdentifier dat1, IndexedIdentifier dat2)
- {
- for( IndexedIdentifier dat : new IndexedIdentifier[]{dat1,dat2} )
- {
- if( dat1.getRowLowerBound()!=null )
- for( DataIdentifier datsub : rGetDataIdentifiers(dat.getRowLowerBound()) )
- if( _bounds._lower.containsKey(datsub.getName()) &&
- !datsub.getName().startsWith(INTERAL_FN_INDEX_ROW) )
- return false;
- if( dat1.getRowUpperBound()!=null )
- for( DataIdentifier datsub : rGetDataIdentifiers(dat.getRowUpperBound()) )
- if( _bounds._lower.containsKey(datsub.getName()) &&
- !datsub.getName().startsWith(INTERAL_FN_INDEX_ROW) )
- return false;
- }
-
- return true;
- }
-
- private boolean isColumnIgnorable(IndexedIdentifier dat1, IndexedIdentifier dat2)
- {
- for( IndexedIdentifier dat : new IndexedIdentifier[]{dat1,dat2} )
- {
- if( dat1.getColLowerBound()!=null )
- for( DataIdentifier datsub : rGetDataIdentifiers(dat.getColLowerBound()) )
- if( _bounds._lower.containsKey(datsub.getName()) &&
- !datsub.getName().startsWith(INTERAL_FN_INDEX_COL) )
- return false;
- if( dat1.getColUpperBound()!=null )
- for( DataIdentifier datsub : rGetDataIdentifiers(dat.getColUpperBound()) )
- if( _bounds._lower.containsKey(datsub.getName()) &&
- !datsub.getName().startsWith(INTERAL_FN_INDEX_COL) )
- return false;
- }
-
- return true;
- }
-
- private List<DataIdentifier> rGetDataIdentifiers(Expression e)
- {
- List<DataIdentifier> ret = new ArrayList<DataIdentifier>();
-
- if( e instanceof DataIdentifier &&
- !(e instanceof FunctionCallIdentifier || e instanceof BuiltinFunctionExpression || e instanceof ParameterizedBuiltinFunctionExpression) )
- {
- ret.add( (DataIdentifier)e );
- }
- else if( e instanceof FunctionCallIdentifier )
- {
- FunctionCallIdentifier fci = (FunctionCallIdentifier)e;
- for( ParameterExpression ee : fci.getParamExprs() )
- ret.addAll(rGetDataIdentifiers( ee.getExpr() ));
- }
- else if(e instanceof BinaryExpression)
- {
- BinaryExpression be = (BinaryExpression) e;
- ret.addAll( rGetDataIdentifiers(be.getLeft()) );
- ret.addAll( rGetDataIdentifiers(be.getRight()) );
- }
- else if(e instanceof BooleanExpression)
- {
- BooleanExpression be = (BooleanExpression) e;
- ret.addAll( rGetDataIdentifiers(be.getLeft()) );
- ret.addAll( rGetDataIdentifiers(be.getRight()) );
- }
- else if(e instanceof BuiltinFunctionExpression)
- {
- BuiltinFunctionExpression be = (BuiltinFunctionExpression) e;
- //disregard meta data ops nrow/ncol (to exclude from candidates)
- if( !((be.getOpCode() == BuiltinFunctionOp.NROW || be.getOpCode() == BuiltinFunctionOp.NCOL)
- && be.getFirstExpr() instanceof DataIdentifier) )
- {
- ret.addAll( rGetDataIdentifiers(be.getFirstExpr()) );
- ret.addAll( rGetDataIdentifiers(be.getSecondExpr()) );
- ret.addAll( rGetDataIdentifiers(be.getThirdExpr()) );
- }
- }
- else if(e instanceof ParameterizedBuiltinFunctionExpression)
- {
- ParameterizedBuiltinFunctionExpression be = (ParameterizedBuiltinFunctionExpression) e;
- for( Expression ee : be.getVarParams().values() )
- ret.addAll( rGetDataIdentifiers(ee) );
- }
- else if(e instanceof RelationalExpression)
- {
- RelationalExpression re = (RelationalExpression) e;
- ret.addAll( rGetDataIdentifiers(re.getLeft()) );
- ret.addAll( rGetDataIdentifiers(re.getRight()) );
- }
-
- return ret;
- }
-
- /**
- *
- * @param sbs
- * @param flag
- * @throws LanguageException
- */
- private void rDetermineBounds( ArrayList<StatementBlock> sbs, boolean flag )
- throws LanguageException
- {
- for( StatementBlock sb : sbs )
- rDetermineBounds(sb, flag);
- }
-
- /**
- * Determines the lower/upper bounds of all nested for/parfor indexes.
- *
- * @param sbs
- * @param flag indicates that method is already in subtree of THIS.
- * @return
- * @throws LanguageException
- */
- private void rDetermineBounds( StatementBlock sb, boolean flag )
- throws LanguageException
- {
- // catch all known for/ parfor bounds
- // (all unknown bounds are assumed to be +-infinity)
-
- for( Statement s : sb._statements )
- {
- boolean lFlag = flag;
- if( s instanceof ParForStatement || (s instanceof ForStatement && CONSERVATIVE_CHECK) ) //incl. for if conservative
- {
- ForStatement fs = (ForStatement)s;
- IterablePredicate ip = fs._predicate;
-
- //checks for position in overall tree
- if( sb==this )
- lFlag = true;
-
- if( lFlag || rIsParent(sb,this) ) //add only if in subtree of this
- {
- //check for internal names
- if( ip.getIterVar()._name.equals( INTERAL_FN_INDEX_ROW )
- || ip.getIterVar()._name.equals( INTERAL_FN_INDEX_COL ))
- {
-
- throw new LanguageException(" The iteration variable must not use the " +
- "internal iteration variable name prefix '"+ip.getIterVar()._name+"'.");
- }
-
- long low = Integer.MIN_VALUE;
- long up = Integer.MAX_VALUE;
- long incr = -1;
-
- if( ip.getFromExpr()instanceof IntIdentifier)
- low = ((IntIdentifier)ip.getFromExpr()).getValue();
- if( ip.getToExpr()instanceof IntIdentifier)
- up = ((IntIdentifier)ip.getToExpr()).getValue();
-
- //NOTE: conservative approach: include all index variables (also from for)
- if( ip.getIncrementExpr() instanceof IntIdentifier )
- incr = ((IntIdentifier)ip.getIncrementExpr()).getValue();
- else
- throw new LanguageException("PARFOR loop dependency analysis: cannot check for dependencies " +
- "because increment expression '"+ip.getIncrementExpr().toString()+"' cannot be normalized.");
-
- _bounds._lower.put(ip.getIterVar()._name, low);
- _bounds._upper.put(ip.getIterVar()._name, up);
- _bounds._increment.put(ip.getIterVar()._name, incr);
- if( lFlag ) //if local (required for constant check)
- _bounds._local.add(ip.getIterVar()._name);
- }
-
- //recursive invocation (but not for nested parfors due to constant check)
- if( !lFlag )
- {
- ArrayList<StatementBlock> tmp = fs.getBody();
- if( tmp != null )
- rDetermineBounds(tmp, lFlag);
- }
- }
- else if( s instanceof ForStatement )
- {
- //recursive invocation
- ArrayList<StatementBlock> tmp = ((ForStatement) s).getBody();
- if( tmp != null )
- rDetermineBounds(tmp, lFlag);
- }
- else if( s instanceof WhileStatement )
- {
- //recursive invocation
- ArrayList<StatementBlock> tmp = ((WhileStatement) s).getBody();
- if( tmp != null )
- rDetermineBounds(tmp, lFlag);
- }
- else if( s instanceof IfStatement )
- {
- //recursive invocation
- ArrayList<StatementBlock> tmp = ((IfStatement) s).getIfBody();
- if( tmp != null )
- rDetermineBounds(tmp, lFlag);
- ArrayList<StatementBlock> tmp2 = ((IfStatement) s).getElseBody();
- if( tmp2 != null )
- rDetermineBounds(tmp2, lFlag);
- }
- else if( s instanceof FunctionStatement )
- {
- //recursive invocation
- ArrayList<StatementBlock> tmp = ((FunctionStatement) s).getBody();
- if( tmp != null )
- rDetermineBounds(tmp, lFlag);
- }
- }
- }
-
- /**
- *
- * @param cParent
- * @param cChild
- * @return
- */
- private boolean rIsParent( ArrayList<StatementBlock> cParent, StatementBlock cChild)
- {
- for( StatementBlock sb : cParent )
- if( rIsParent(sb, cChild) )
- return true;
-
- return false;
- }
-
- /**
- *
- * @param cParent
- * @param cChild
- * @return
- */
- private boolean rIsParent( StatementBlock cParent, StatementBlock cChild)
- {
- boolean ret = false;
-
- if( cParent == cChild )
- {
- ret = true;
- }
- else
- {
- for( Statement s : cParent.getStatements() )
- {
- //check all the complex control flow constructs
- if( s instanceof ForStatement ) //for, parfor
- {
- ret = rIsParent( ((ForStatement) s).getBody(), cChild );
- }
- else if( s instanceof WhileStatement )
- {
- ret = rIsParent( ((WhileStatement) s).getBody(), cChild );
- }
- else if( s instanceof IfStatement )
- {
- ret = rIsParent( ((IfStatement) s).getIfBody(), cChild );
- ret |= rIsParent( ((IfStatement) s).getElseBody(), cChild );
- }
-
- //early return if already found
- if( ret )
- break;
- }
- }
-
- return ret;
- }
-
- /**
- * Runs a combination of GCD and Banerjee test for a two potentially conflicting
- * data identifiers. See below for a detailed explanation.
- *
- * NOTE: simply enumerating all combinations of iteration variable values and probing for
- * duplicates is not applicable due to (1) arbitrary nested program blocks with potentially
- * dynamic lower, upper, and increment expressions, and (2) therefore potentially large
- * overheads in the general case.
- *
- * @param dat1
- * @param dat2
- * @return
- * @throws LanguageException
- */
- private boolean runBanerjeeGCDTest(DataIdentifier dat1, DataIdentifier dat2)
- throws LanguageException
- {
- /* The GCD (greatest common denominator) and the Banerjee test are two commonly used tests
- * for determining loop-carried dependencies. Both rely on (1) linear index expressions of the
- * form y = a + bx, where x is the loop index variable, and (2) conservative approaches that
- * guarantee no false negatives (no missed dependencies) but possibly false positives. The GCD
- * test probes for integer solutions without bounds, while the Banerjee test probes for real
- * solutions with bounds.
- *
- * We use a combination of both:
- * - the GCD test checks if dependencies are possible
- * - the Banerjee test checks if those dependencies may arise within the given bounds
- *
- * NOTES:
- * - #1 possible false positives may arise if there is a real solution within the bounds
- * and an integer solution outside the bounds. This will lead to a detected dependencies
- * although no integer solution within the bounds exists.
- * - #2 for the sake of simplicity, we do not distinguish between anti and data dependencies,
- * although possible in general
- * - more advanced tests than GCD and Banerjee available (e.g., with symbolic checking for
- * non-linear functions) but this is a tradeoff between number of false positives and overhead
- */
-
- LOG.trace("PARFOR: runBanerjeeGCDCheck.");
-
- boolean ret = true; //anti or data dependency
-
- //Step 1: analyze index expressions and transform them into linear functions
- LinearFunction f1 = getLinearFunction(dat1);
- LinearFunction f2 = getLinearFunction(dat2);
- forceConsistency(f1,f2);
-
- LOG.trace("PARFOR: f1: " + f1.toString());
- LOG.trace("PARFOR: f2: " + f2.toString());
-
- ///////
- //Step 2: run GCD Test
- ///////
- long lgcd = f1._b[0];
- for( int i=1; i<f1._b.length; i++ )
- lgcd = determineGCD( lgcd, f1._b[i] );
- for( int i=0; i<f2._b.length; i++ )
- lgcd = determineGCD( lgcd, f2._b[i] );
-
- if( (Math.abs(f1._a-f2._a) % lgcd) != 0 ) //if GCD divides the intercepts
- {
- //no integer solution exists -> no dependency
- ret = false;
- }
-
- LOG.trace("PARFOR: GCD result: "+ret);
-
- if( !CONSERVATIVE_CHECK && ret ) //only if not already no dependency
- {
- //NOTE: cases both and none negligible already covered (constant check, general case)
- boolean ixid = (dat1 instanceof IndexedIdentifier && dat2 instanceof IndexedIdentifier);
- boolean ignoreRow = ixid && isRowIgnorable((IndexedIdentifier)dat1, (IndexedIdentifier)dat2);
- boolean ignoreCol = ixid && isColumnIgnorable((IndexedIdentifier)dat1, (IndexedIdentifier)dat2);
-
- LinearFunction f1p = null, f2p = null;
- if( ignoreRow )
- {
- f1p = getColLinearFunction(dat1);
- f2p = getColLinearFunction(dat2);
- }
- if( ignoreCol )
- {
- f1p = getRowLinearFunction(dat1);
- f2p = getRowLinearFunction(dat2);
- }
-
- LOG.trace("PARFOR: f1p: "+((f1p==null)?"null":f1p.toString()));
- LOG.trace("PARFOR: f2p: "+((f2p==null)?"null":f2p.toString()));
-
- if( f1p!=null && f2p!=null )
- {
- forceConsistency(f1p, f2p);
-
- long lgcd2 = f1p._b[0];
- for( int i=1; i<f1p._b.length; i++ )
- lgcd2 = determineGCD( lgcd2, f1p._b[i] );
- for( int i=0; i<f2p._b.length; i++ )
- lgcd2 = determineGCD( lgcd2, f2p._b[i] );
-
- if( (Math.abs(f1p._a-f2p._a) % lgcd2) != 0 ) //if GCD divides the intercepts
- {
- //no integer solution exists -> no dependency
- ret = false;
- }
-
- LOG.trace("PARFOR: GCD result: "+ret);
- }
- }
-
-
- ///////
- //Step 3: run Banerjee Test
- ///////
- if( ret ) //only if GCD found possible dependencies
- {
- long lintercept = f2._a - f1._a;
-
- //determining anti/data dependencies
- long lmax=0;
- long lmin=0;
-
- //min/max bound
- int len = Math.max(f1._b.length, f2._b.length);
- for( int i=0; i<len; i++ )
- {
- String var=(f1._b.length>i) ? f1._vars[i] : f2._vars[i];
-
- //get lower and upper bound for specific var or internal var
- long lower = _bounds._lower.get(var); //bounds equal for f1 and f2
- long upper = _bounds._upper.get(var);
-
- //max bound
- if( f1._b.length>i )
- {
- if( f1._b[i]>0 )
- lmax += f1._b[i]*upper;
- else
- lmax += f1._b[i]*lower;
- }
- if( f2._b.length>i )
- {
- if( f2._b[i]>0 )
- lmax -= f2._b[i]*lower;
- else
- lmax -= f2._b[i]*upper;
- }
-
- //min bound (unequal indexes)
- if( f1._b.length>i )
- {
- if( f1._b[i]>0 )
- lmin += f1._b[i]*lower;
- else
- lmin += f1._b[i]*upper;
- }
- if( f2._b.length>i )
- {
- if( f2._b[i]>0 )
- lmin -= f2._b[i]*upper;
- else
- lmin -= f2._b[i]*lower;
- }
- }
-
- LOG.trace("PARFOR: Banerjee lintercept " + lintercept);
- LOG.trace("PARFOR: Banerjee lmax " + lmax);
- LOG.trace("PARFOR: Banerjee lmin " + lmin);
-
-
- if( !(lmin <= lintercept && lintercept <= lmax) || lmin==lmax )
- {
- //dependency not within the bounds of the arrays
- ret = false;
- }
-
- LOG.trace("PARFOR: Banerjee result: "+ret);
- }
-
- return ret;
- }
-
- /**
- * Runs a constant check for a single data identifier (target of assignment). If constant, then every
- * iteration writes to the same cell.
- *
- * @param dat1
- * @return
- * @throws LanguageException
- */
- private boolean runConstantCheck(DataIdentifier dat1)
- throws LanguageException
- {
- LOG.trace("PARFOR: runConstantCheck.");
-
- boolean ret = true; //data dependency to itself
- LinearFunction f1 = getLinearFunction(dat1);
- if( f1 == null )
- return true; //dependency
-
- LOG.trace("PARFOR: f1: "+f1.toString());
-
- // no output dependency to itself if no index access will happen twice
- // hence we check for: (all surrounding indexes are used by f1 and all intercepts != 0 )
- boolean gcheck=true;
- for( String var : _bounds._local ) //check only local, nested checked from parent
- {
- if( var.startsWith(INTERAL_FN_INDEX_ROW)
- || var.startsWith(INTERAL_FN_INDEX_COL))
- {
- continue; //skip internal vars for range indexing
- }
-
- boolean lcheck=false;
- for( int i=0; i<f1._vars.length; i++ )
- if( var.equals(f1._vars[i]) )
- if( f1._b[i] != 0 )
- lcheck = true;
- if( !lcheck )
- {
- gcheck=false;
- break;
- }
- }
-
- if( gcheck ) // output dependencies impossible
- ret = false;
-
- return ret;
- }
-
- /**
- * Runs an equality check for two data identifiers. If equal, there there are no
- * inter-iteration (loop-carried) but only intra-iteration dependencies.
- *
- * @param dat1
- * @param dat2
- * @return
- * @throws LanguageException
- */
- private boolean runEqualsCheck(DataIdentifier dat1, DataIdentifier dat2)
- throws LanguageException
- {
- LOG.trace("PARFOR: runEqualsCheck.");
-
- //check if both data identifiers of same type
- if(dat1 instanceof IndexedIdentifier != dat2 instanceof IndexedIdentifier)
- return false;
-
- //general case function comparison
- boolean ret = true; //true if equal index functions
- LinearFunction f1 = getLinearFunction(dat1);
- LinearFunction f2 = getLinearFunction(dat2);
- forceConsistency(f1, f2);
- ret = f1.equals(f2);
-
- LOG.trace("PARFOR: f1: " + f1.toString());
- LOG.trace("PARFOR: f2: " + f2.toString());
- LOG.trace("PARFOR: (f1==f2): " + ret);
-
- //additional check if cols/rows could be ignored
- if( !CONSERVATIVE_CHECK && !ret ) //only if not already equal
- {
- //NOTE: cases both and none negligible already covered (constant check, general case)
- boolean ixid = (dat1 instanceof IndexedIdentifier && dat2 instanceof IndexedIdentifier);
- boolean ignoreRow = ixid && isRowIgnorable((IndexedIdentifier)dat1, (IndexedIdentifier)dat2);
- boolean ignoreCol = ixid && isColumnIgnorable((IndexedIdentifier)dat1, (IndexedIdentifier)dat2);
-
- LinearFunction f1p = null, f2p = null;
- if( ignoreRow )
- {
- f1p = getColLinearFunction(dat1);
- f2p = getColLinearFunction(dat2);
- }
- if( ignoreCol )
- {
- f1p = getRowLinearFunction(dat1);
- f2p = getRowLinearFunction(dat2);
- }
-
- if( f1p!=null && f2p!=null )
- {
- forceConsistency(f1p, f2p);
- ret = f1p.equals(f2p);
-
- LOG.trace("PARFOR: f1p: " + f1p.toString());
- LOG.trace("PARFOR: f2p: " + f2p.toString());
- LOG.trace("PARFOR: (f1p==f2p): " + ret);
- }
- }
-
- return ret;
- }
-
- /**
- * This is the Euclid's algorithm for GCD (greatest common denominator),
- * required for the GCD test.
- *
- * @param a
- * @param b
- * @return
- */
- private long determineGCD(long a, long b)
- {
- if (b==0)
- return a;
- else
- return determineGCD(b, a % b);
- }
-
- /**
- * Creates or reuses a linear function for a given data identifier, where identifiers with equal
- * names and matrix subscripts result in exactly the same linear function.
- *
- * @param dat
- * @return
- * @throws LanguageException
- */
- private LinearFunction getLinearFunction(DataIdentifier dat)
- throws LanguageException
- {
- /* Notes:
- * - Currently, this function supports 2dim matrix subscripts with arbitrary linear functions
- * however, this could be extended to d-dim if necessary
- * - Trick for range indexing: introduce a pseudo index variable with lower and upper according to
- * the index range (e.g., [1:4,...]) or matrix dimensionality (e.g., [:,...]). This allows us to
- * apply existing tests even for range indexing (multi-value instead of single-value functions)
- */
-
- LinearFunction out = null;
-
- if( ! (dat instanceof IndexedIdentifier ) ) //happens if matrix is now used as scalar
- return new LinearFunction(0,0,dat.getName());
-
- IndexedIdentifier idat = (IndexedIdentifier) dat;
-
- if( USE_FN_CACHE )
- {
- out = _fncache.get( getFunctionID(idat) );
- if( out != null )
- return out;
- }
-
- Expression sub1 = idat.getRowLowerBound();
- Expression sub2 = idat.getColLowerBound();
-
- //parse row expressions
- try
- {
- //loop index or constant (default case)
- if( idat.getRowLowerBound()!=null && idat.getRowUpperBound()!=null &&
- idat.getRowLowerBound() == idat.getRowUpperBound() )
- {
- if( sub1 instanceof IntIdentifier )
- out = new LinearFunction(((IntIdentifier)sub1).getValue(), 0, null);
- else if( sub1 instanceof DataIdentifier )
- out = new LinearFunction(0, 1, ((DataIdentifier)sub1)._name);
- else
- out = rParseBinaryExpression((BinaryExpression)sub1);
-
- if( !CONSERVATIVE_CHECK )
- if(out.hasNonIndexVariables())
- {
- String id = INTERAL_FN_INDEX_ROW+_idSeqfn.getNextID();
- out = new LinearFunction(0, 1L, id);
-
- _bounds._lower.put(id, 1L);
- _bounds._upper.put(id, _vsParent.getVariable(idat._name).getDim1()); //row dim
- _bounds._increment.put(id, 1L);
- }
- }
- else //range indexing
- {
- Expression sub1a = sub1;
- Expression sub1b = idat.getRowUpperBound();
-
- String id = INTERAL_FN_INDEX_ROW+_idSeqfn.getNextID();
- out = new LinearFunction(0, 1L, id);
-
- if( sub1a == null && sub1b == null //: operator
- || !(sub1a instanceof IntIdentifier) || !(sub1b instanceof IntIdentifier) ) //for robustness
- {
- _bounds._lower.put(id, 1L);
- _bounds._upper.put(id, _vsParent.getVariable(idat._name).getDim1()); //row dim
- _bounds._increment.put(id, 1L);
- }
- else if( sub1a instanceof IntIdentifier && sub1b instanceof IntIdentifier )
- {
- _bounds._lower.put(id, ((IntIdentifier)sub1a).getValue());
- _bounds._upper.put(id, ((IntIdentifier)sub1b).getValue());
- _bounds._increment.put(id, 1L);
- }
- else
- {
- out = null;
- }
- }
-
- //scale row function 'out' with col dimensionality
- long colDim = _vsParent.getVariable(idat._name).getDim2();
- if( colDim > 0 )
- {
- out.scale( colDim );
- }
- else
- {
- //NOTE: we could mark sb for deferred validation and evaluate on execute (see ParForProgramBlock)
- LOG.debug("PARFOR: Warning - matrix dimensionality of '"+idat._name+"' unknown, cannot scale linear functions.");
- }
- }
- catch(Exception ex)
- {
- LOG.debug("PARFOR: Unable to parse MATRIX subscript expression for '"+String.valueOf(sub1)+"'.", ex);
- out = null; //let dependency analysis fail
- }
-
- //parse col expression and merge functions
- if( out!=null )
- {
- try
- {
- LinearFunction tmpOut = null;
-
- //loop index or constant (default case)
- if( idat.getColLowerBound()!=null && idat.getColUpperBound()!=null &&
- idat.getColLowerBound() == idat.getColUpperBound() )
- {
- if( sub2 instanceof IntIdentifier )
- out.addConstant( ((IntIdentifier)sub2).getValue() );
- else if( sub2 instanceof DataIdentifier )
- tmpOut = new LinearFunction(0, 1, ((DataIdentifier)sub2)._name) ;
- else
- tmpOut = rParseBinaryExpression((BinaryExpression)sub2);
-
- if( !CONSERVATIVE_CHECK )
- if(tmpOut!=null && tmpOut.hasNonIndexVariables())
- {
- String id = INTERAL_FN_INDEX_COL+_idSeqfn.getNextID();
- tmpOut = new LinearFunction(0, 1L, id);
- _bounds._lower.put(id, 1l);
- _bounds._upper.put(id, _vsParent.getVariable(idat._name).getDim2()); //col dim
- _bounds._increment.put(id, 1L);
- }
- }
- else //range indexing
- {
- Expression sub2a = sub2;
- Expression sub2b = idat.getColUpperBound();
-
- String id = INTERAL_FN_INDEX_COL+_idSeqfn.getNextID();
- tmpOut = new LinearFunction(0, 1L, id);
-
- if( sub2a == null && sub2b == null //: operator
- || !(sub2a instanceof IntIdentifier) || !(sub2b instanceof IntIdentifier) ) //for robustness
- {
- _bounds._lower.put(id, 1L);
- _bounds._upper.put(id, _vsParent.getVariable(idat._name).getDim2()); //col dim
- _bounds._increment.put(id, 1L);
- }
- else if( sub2a instanceof IntIdentifier && sub2b instanceof IntIdentifier )
- {
- _bounds._lower.put(id, ((IntIdentifier)sub2a).getValue());
- _bounds._upper.put(id, ((IntIdentifier)sub2b).getValue());
- _bounds._increment.put(id, 1L);
- }
- else
- {
- out = null;
- }
- }
-
- //final merge of row and col functions
- if( tmpOut != null )
- out.addFunction(tmpOut);
- }
- catch(Exception ex)
- {
- LOG.debug("PARFOR: Unable to parse MATRIX subscript expression for '"+String.valueOf(sub2)+"'.", ex);
- out = null; //let dependency analysis fail
- }
- }
-
- //post processing after creation
- if( out != null )
- {
- //cleanup and verify created function; raise exceptions if needed
- cleanupFunction(out);
- verifyFunction(out);
-
- // pseudo loop normalization of functions (incr=1, from=1 not necessary due to Banerjee)
- // (precondition for GCD test)
- if( NORMALIZE )
- {
- int index=0;
- for( String var : out._vars )
- {
- long low = _bounds._lower.get(var);
- long up = _bounds._upper.get(var);
- long incr = _bounds._increment.get(var);
- if( incr < 0 || 1 < incr ) //does never apply to internal (artificial) vars
- {
- out.normalize(index,low,incr); // normalize linear functions
- _bounds._upper.put(var,(long)Math.ceil(((double)up)/incr)); // normalize upper bound
- }
- index++;
- }
- }
-
- //put into cache
- if( USE_FN_CACHE )
- {
- _fncache.put( getFunctionID(idat), out );
- }
- }
-
- return out;
- }
-
- private LinearFunction getRowLinearFunction(DataIdentifier dat)
- throws LanguageException
- {
- //NOTE: would require separate function cache, not realized due to inexpensive operations
-
- LinearFunction out = null;
- IndexedIdentifier idat = (IndexedIdentifier) dat;
- Expression sub1 = idat.getRowLowerBound();
-
- try
- {
- //loop index or constant (default case)
- if( idat.getRowLowerBound()!=null && idat.getRowUpperBound()!=null &&
- idat.getRowLowerBound() == idat.getRowUpperBound() )
- {
- if( sub1 instanceof IntIdentifier )
- out = new LinearFunction(((IntIdentifier)sub1).getValue(), 0, null);
- else if( sub1 instanceof DataIdentifier )
- out = new LinearFunction(0, 1, ((DataIdentifier)sub1)._name); //never use public members
- else
- out = rParseBinaryExpression((BinaryExpression)sub1);
- }
- }
- catch(Exception ex)
- {
- LOG.debug("PARFOR: Unable to parse MATRIX subscript expression for '"+String.valueOf(sub1)+"'.", ex);
- out = null; //let dependency analysis fail
- }
-
- //post processing after creation
- if( out != null )
- {
- //cleanup and verify created function; raise exceptions if needed
- cleanupFunction(out);
- verifyFunction(out);
- }
-
- return out;
- }
-
- private LinearFunction getColLinearFunction(DataIdentifier dat)
- throws LanguageException
- {
- //NOTE: would require separate function cache, not realized due to inexpensive operations
-
- LinearFunction out = null;
- IndexedIdentifier idat = (IndexedIdentifier) dat;
- Expression sub1 = idat.getColLowerBound();
-
- try
- {
- //loop index or constant (default case)
- if( idat.getColLowerBound()!=null && idat.getColUpperBound()!=null &&
- idat.getColLowerBound() == idat.getColUpperBound() )
- {
- if( sub1 instanceof IntIdentifier )
- out = new LinearFunction(((IntIdentifier)sub1).getValue(), 0, null);
- else if( sub1 instanceof DataIdentifier )
- out = new LinearFunction(0, 1, ((DataIdentifier)sub1)._name); //never use public members
- else
- out = rParseBinaryExpression((BinaryExpression)sub1);
- }
- }
- catch(Exception ex)
- {
- LOG.debug("PARFOR: Unable to parse MATRIX subscript expression for '"+String.valueOf(sub1)+"'.", ex);
- out = null; //let dependency analysis fail
- }
-
- //post processing after creation
- if( out != null )
- {
- //cleanup and verify created function; raise exceptions if needed
- cleanupFunction(out);
- verifyFunction(out);
- }
-
- return out;
- }
-
- /**
- * Creates a functionID for a given data identifier (mainly used for caching purposes),
- * where data identifiers with equal name and matrix subscripts results in equal
- * functionIDs.
- *
- * @param dat
- * @return
- */
- private String getFunctionID( IndexedIdentifier dat )
- {
- /* note: using dat.hashCode can be different for same functions,
- * hence, we use a custom String ID
- */
-
- IndexedIdentifier idat = (IndexedIdentifier) dat;
- Expression ex1a = idat.getRowLowerBound();
- Expression ex1b = idat.getRowUpperBound();
- Expression ex2a = idat.getColLowerBound();
- Expression ex2b = idat.getColUpperBound();
-
- StringBuilder sb = new StringBuilder();
- sb.append(String.valueOf(ex1a));
- sb.append(',');
- sb.append(String.valueOf(ex1b));
- sb.append(',');
- sb.append(String.valueOf(ex2a));
- sb.append(',');
- sb.append(String.valueOf(ex2b));
-
- return sb.toString();
- }
-
-
-
- /**
- * Removes all zero intercepts created by recursive computation.
- *
- * @param f1
- */
- private void cleanupFunction( LinearFunction f1 )
- {
- for( int i=0; i<f1._b.length; i++ )
- {
- if( f1._vars[i]==null )
- {
- f1.removeVar(i);
- i--;
- continue;
- }
- }
- }
-
- /**
- * Simply verification check of created linear functions, mainly used for
- * robustness purposes.
- *
- * @param f1
- * @throws LanguageException
- */
- private void verifyFunction(LinearFunction f1)
- throws LanguageException
- {
- //check for required form of linear functions
- if( f1 == null || f1._b.length != f1._vars.length )
- {
- if( LOG.isTraceEnabled() && f1!=null )
- LOG.trace("PARFOR: f1: "+f1.toString());
-
- throw new LanguageException("PARFOR loop dependency analysis: " +
- "MATRIX subscripts are not in linear form (a0 + a1*x).");
- }
-
- //check all function variables to be index variables
- for( String var : f1._vars )
- {
- if( !_bounds._lower.containsKey(var) )
- {
- LOG.trace("PARFOR: not allowed variable in matrix subscript: "+var);
- throw new LanguageException("PARFOR loop dependency analysis: " +
- "MATRIX subscripts use non-index variables.");
- }
- }
- }
-
- /**
- * Tries to obtain consistent linear functions by forcing the same variable ordering for
- * efficient comparison: f2 is modified in a way that it matches the sequence of variables in f1.
- *
- * @param f1
- * @param f2
- */
- private void forceConsistency(LinearFunction f1, LinearFunction f2)
- {
- boolean warn = false;
-
- for( int i=0; i<f1._b.length; i++ )
- {
- if( f2._b.length<(i+1) )
- break;
-
- if( !f1._vars[i].equals(f2._vars[i])
- &&!(f1._vars[i].startsWith(INTERAL_FN_INDEX_ROW) && f2._vars[i].startsWith(INTERAL_FN_INDEX_ROW))
- &&!(f1._vars[i].startsWith(INTERAL_FN_INDEX_COL) && f2._vars[i].startsWith(INTERAL_FN_INDEX_COL)))
- {
- boolean exchange = false;
- //scan
- for( int j=i+1; j<f2._b.length; j++ )
- if( f1._vars[i].equals(f2._vars[j])
- ||(f1._vars[i].startsWith(INTERAL_FN_INDEX_ROW) && f2._vars[j].startsWith(INTERAL_FN_INDEX_ROW))
- ||(f1._vars[i].startsWith(INTERAL_FN_INDEX_COL) && f2._vars[j].startsWith(INTERAL_FN_INDEX_COL)) )
- {
- //exchange
- long btmp = f2._b[i];
- String vartmp = f2._vars[i];
- f2._b[i] = f2._b[j];
- f2._vars[i] = f2._vars[j];
- f2._b[j] = btmp;
- f2._vars[j] = vartmp;
- exchange = true;
- }
- if( !exchange )
- warn = true;
- }
- }
-
-
- if( warn && LOG.isTraceEnabled() )
- LOG.trace( "PARFOR: Warning - index functions f1 and f2 cannot be made consistent." );
- }
-
- /**
- * Recursively creates a linear function for a single BinaryExpression, where PLUS, MINUS, MULT
- * are allowed as operators.
- *
- * @param be
- * @return
- * @throws LanguageException
- */
- private LinearFunction rParseBinaryExpression(BinaryExpression be)
- throws LanguageException
- {
- LinearFunction ret = null;
- Expression l = be.getLeft();
- Expression r = be.getRight();
-
- if( be.getOpCode() == BinaryOp.PLUS )
- {
- //parse binary expressions
- if( l instanceof BinaryExpression)
- {
- ret = rParseBinaryExpression((BinaryExpression) l);
- Long cvalR = parseLongConstant(r);
- if( cvalR != null )
- ret.addConstant(cvalR);
- else
- return null;
- }
- else if (r instanceof BinaryExpression)
- {
- ret = rParseBinaryExpression((BinaryExpression) r);
- Long cvalL = parseLongConstant(l);
- if( cvalL != null )
- ret.addConstant(cvalL);
- else
- return null;
- }
- else // atomic case
- {
- Long cvalL = parseLongConstant(l);
- Long cvalR = parseLongConstant(r);
- if( cvalL != null )
- ret = new LinearFunction(cvalL,1,((DataIdentifier)r)._name);
- else if( cvalR != null )
- ret = new LinearFunction(cvalR,1,((DataIdentifier)l)._name);
- else
- return null; //let dependency analysis fail
- }
- }
- else if( be.getOpCode() == BinaryOp.MINUS )
- {
- //parse binary expressions
- if( l instanceof BinaryExpression)
- {
- ret = rParseBinaryExpression((BinaryExpression) l);
- //change to plus
- Long cvalR = parseLongConstant(r);
- ret.addConstant(cvalR*(-1));
- }
- else if (r instanceof BinaryExpression)
- {
- ret = rParseBinaryExpression((BinaryExpression) r);
- //change to plus
- ret._a*=(-1);
- for( int i=0; i<ret._b.length; i++ )
- ret._b[i]*=(-1);
- Long cvalL = parseLongConstant(l);
- ret.addConstant(cvalL);
- }
- else // atomic case
- {
- //change everything to plus
- Long cvalL = parseLongConstant(l);
- Long cvalR = parseLongConstant(r);
- if( cvalL != null )
- ret = new LinearFunction(cvalL,-1,((DataIdentifier)r)._name);
- else if( cvalR != null )
- ret = new LinearFunction(cvalR*(-1),1,((DataIdentifier)l)._name);
- else
- return null; //let dependency analysis fail
- }
- }
- else if( be.getOpCode() == BinaryOp.MULT )
- {
- //NOTE: no recursion for MULT expressions
-
- //atomic case
- Long cvalL = parseLongConstant(l);
- Long cvalR = parseLongConstant(r);
-
- if( cvalL != null )
- ret = new LinearFunction(0, cvalL,((DataIdentifier)r)._name);
- else if( cvalR != null )
- ret = new LinearFunction(0, cvalR,((DataIdentifier)l)._name);
- else
- return null; //let dependency analysis fail
- }
- else
- return null; //let dependency analysis fail
-
- return ret;
- }
-
- /**
- *
- * @param expr
- * @return
- */
- private Long parseLongConstant(Expression expr)
- {
- Long ret = null;
-
- if( expr instanceof IntIdentifier ) {
- ret = ((IntIdentifier) expr).getValue();
- }
- else if( expr instanceof DoubleIdentifier ) {
- double tmp = ((DoubleIdentifier) expr).getValue();
- //ensure double represent an integer number
- if( tmp == Math.floor(tmp) )
- ret = UtilFunctions.toLong(tmp);
- }
-
- return ret;
- }
-
- /**
- * Helper class for representing a single candidate.
- *
- */
- private static class Candidate
- {
- String _var; // variable name
- //Integer _pos; // statement position in parfor (can be used for distinguishing anti/data dep)
- DataIdentifier _dat; // _var data identifier
- }
-
- /**
- * Helper class for representing all lower, upper bounds of (potentially nested)
- * loop constructs.
- *
- */
- private static class Bounds
- {
- HashMap<String, Long> _lower = new HashMap<String, Long>();
- HashMap<String, Long> _upper = new HashMap<String, Long>();
- HashMap<String, Long> _increment = new HashMap<String, Long>();
-
- //contains all local variable names (subset of lower/upper/incr sets)
- HashSet<String> _local = new HashSet<String>();
- }
-
- /**
- * Helper class for representing linear functions of matrix subscripts.
- * The allowed form is 'y = a + b1x1 + ... = bnxn', which is required by
- * the applied GCD and Banerjee tests.
- *
- */
- private class LinearFunction
- {
- long _a; // intercept
- long[] _b; // slopes
- String[] _vars; // b variable names
-
- LinearFunction( long a, long b, String name )
- {
- _a = a;
- _b = new long[1];
- _b[0] = b;
- _vars = new String[1];
- _vars[0] = name;
- }
-
- public void addConstant(long value)
- {
- _a += value;
- }
-
- public void addFunction( LinearFunction f2)
- {
- _a = _a + f2._a;
-
- long[] tmpb = new long[_b.length+f2._b.length];
- System.arraycopy( _b, 0, tmpb, 0, _b.length );
- System.arraycopy( f2._b, 0, tmpb, _b.length, f2._b.length );
- _b = tmpb;
-
- String[] tmpvars = new String[_vars.length+f2._vars.length];
- System.arraycopy( _vars, 0, tmpvars, 0, _vars.length );
- System.arraycopy( f2._vars, 0, tmpvars, _vars.length, f2._vars.length );
- _vars = tmpvars;
- }
-
- public void removeVar( int i )
- {
- long[] tmpb = new long[_b.length-1];
-
- System.arraycopy( _b, 0, tmpb, 0, i );
- System.arraycopy( _b, i+1, tmpb, i, _b.length-i-1 );
- _b = tmpb;
-
- String[] tmpvars = new String[_vars.length-1];
- System.arraycopy( _vars, 0, tmpvars, 0, i );
- System.arraycopy( _vars, i+1, tmpvars, i, _vars.length-i-1 );
- _vars = tmpvars;
- }
-
- public void scale( long scale )
- {
- _a *= scale; //-1 because indexing starts at 1
-
- for( int i=0; i<_b.length; i++ )
- if( _b[i] != 0 )
- _b[i] *= scale;
- }
-
- public void normalize(int index, long lower, long increment)
- {
- _a -= (_b[index] * lower);
- _b[index] *= increment;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("(");
- sb.append(_a);
- sb.append(") + ");
- sb.append("(");
- for( int i=0; i<_b.length; i++ )
- {
- if( i>0 )
- sb.append("+");
- sb.append("(");
- sb.append(_b[i]);
- sb.append(" * ");
- sb.append(_vars[i]);
- sb.append(")");
- }
- sb.append(")");
-
- return sb.toString();
- }
-
- @Override
- public boolean equals( Object o2 )
- {
- if( o2 == null || !(o2 instanceof LinearFunction) )
- return false;
-
- LinearFunction f2 = (LinearFunction)o2;
- boolean ret = true;
- ret &= ( _a == f2._a );
- ret &= ( _b.length == f2._b.length );
-
- if( ret )
- {
- for( int i=0; i<_b.length; i++ )
- {
- ret &= (_b[i] == f2._b[i] );
- ret &= (_vars[i].equals(f2._vars[i])
- ||(_vars[i].startsWith(INTERAL_FN_INDEX_ROW) && f2._vars[i].startsWith(INTERAL_FN_INDEX_ROW))
- ||(_vars[i].startsWith(INTERAL_FN_INDEX_COL) && f2._vars[i].startsWith(INTERAL_FN_INDEX_COL)) ) ;
- }
- }
-
- return ret;
- }
-
- @Override
- public int hashCode()
- {
- //use identity hash code
- return super.hashCode();
- }
-
- public boolean hasNonIndexVariables()
- {
- boolean ret = false;
- for( String var : _vars )
- if( var!=null && !_bounds._lower.containsKey(var) )
- {
- ret = true;
- break;
- }
-
- return ret;
- }
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ParameterExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParameterExpression.java b/src/main/java/com/ibm/bi/dml/parser/ParameterExpression.java
deleted file mode 100644
index 2745384..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ParameterExpression.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-public class ParameterExpression
-{
-
-
- private Expression _expr;
- private String _name;
-
- public ParameterExpression(String name, Expression val){
- _name = name;
- _expr = val;
- }
-
- public String getName(){
- return _name;
- }
-
- public Expression getExpr(){
- return _expr;
- }
-
- public void setName(String name){
- _name = name;
- }
-
- public void setExpr(Expression expr){
- _expr = expr;
- }
-
- @Override
- public String toString(){
- return _name + "=" + _expr;
- }
-
-}
[12/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidator.java b/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidator.java
deleted file mode 100644
index 88b6138..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidator.java
+++ /dev/null
@@ -1,2107 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.python;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.ErrorNode;
-import org.antlr.v4.runtime.tree.TerminalNode;
-import org.apache.commons.lang.StringUtils;
-
-import com.ibm.bi.dml.parser.AssignmentStatement;
-import com.ibm.bi.dml.parser.BinaryExpression;
-import com.ibm.bi.dml.parser.BooleanExpression;
-import com.ibm.bi.dml.parser.BooleanIdentifier;
-import com.ibm.bi.dml.parser.BuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ConditionalPredicate;
-import com.ibm.bi.dml.parser.ConstIdentifier;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DoubleIdentifier;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.Expression.DataOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.FunctionCallIdentifier;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.ImportStatement;
-import com.ibm.bi.dml.parser.IndexedIdentifier;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.IterablePredicate;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.MultiAssignmentStatement;
-import com.ibm.bi.dml.parser.OutputStatement;
-import com.ibm.bi.dml.parser.ParForStatement;
-import com.ibm.bi.dml.parser.ParameterExpression;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.PathStatement;
-import com.ibm.bi.dml.parser.PrintStatement;
-import com.ibm.bi.dml.parser.RelationalExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.WhileStatement;
-//import com.ibm.bi.dml.parser.antlr4.ExpressionInfo;
-//import com.ibm.bi.dml.parser.antlr4.StatementInfo;
-import com.ibm.bi.dml.parser.python.PydmlParser.AddSubExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.AssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.AtomicExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanAndExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanNotExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BooleanOrExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.BuiltinFunctionExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.CommandlineParamExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.CommandlinePositionExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstDoubleIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstFalseExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstIntIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstStringIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ConstTrueExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.DataIdExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.DataIdentifierContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ExternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ForStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallMultiAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IfStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IfdefAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IgnoreNewLineContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ImportStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IndexedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.InternalFunctionDefExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IterablePredicateColonExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.IterablePredicateSeqExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.MatrixDataTypeCheckContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.Ml_typeContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ModIntDivExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.MultDivExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParForStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PathStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PmlprogramContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.PowerExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.RelationalExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.SimpleDataIdentifierExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StrictParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.StrictParameterizedKeyValueStringContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.TypedArgNoAssignContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.UnaryExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ValueDataTypeCheckContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.WhileStatementContext;
-
-public class PydmlSyntacticValidator implements PydmlListener
-{
- private PydmlSyntacticValidatorHelper helper = null;
-
- private String _workingDir = "."; //current working directory
- private String _currentPath = null; //current file path
- private HashMap<String,String> argVals = null;
-
- public PydmlSyntacticValidator(PydmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
- this.helper = helper;
- this.argVals = argVals;
-
- _currentPath = currentPath;
- }
-
- // Functions we have to implement but don't really need it
- @Override
- public void enterAddSubExpression(AddSubExpressionContext ctx) { }
- @Override
- public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
- @Override
- public void enterAtomicExpression(AtomicExpressionContext ctx) { }
- @Override
- public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
- @Override
- public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
- @Override
- public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
- @Override
- public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
- @Override
- public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
- @Override
- public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
- @Override
- public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
- @Override
- public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
- @Override
- public void enterDataIdExpression(DataIdExpressionContext ctx) { }
-
- @Override
- public void enterIgnoreNewLine(IgnoreNewLineContext ctx) { }
- @Override
- public void enterPmlprogram(PmlprogramContext ctx) { }
- @Override
- public void exitPmlprogram(PmlprogramContext ctx) { }
-
- @Override
- public void enterEveryRule(ParserRuleContext arg0) {
- if(arg0 instanceof StatementContext) {
- if(((StatementContext) arg0).info == null) {
- ((StatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof FunctionStatementContext) {
- if(((FunctionStatementContext) arg0).info == null) {
- ((FunctionStatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof ExpressionContext) {
- if(((ExpressionContext) arg0).info == null) {
- ((ExpressionContext) arg0).info = new ExpressionInfo();
- }
- }
- if(arg0 instanceof DataIdentifierContext) {
- if(((DataIdentifierContext) arg0).dataInfo == null) {
- ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
- }
- }
- }
- @Override
- public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterForStatement(ForStatementContext ctx) {}
- @Override
- public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
- @Override
- public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
- @Override
- public void enterIfStatement(IfStatementContext ctx) { }
- @Override
- public void enterImportStatement(ImportStatementContext ctx) { }
- @Override
- public void enterIndexedExpression(IndexedExpressionContext ctx) { }
- @Override
- public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterMl_type(Ml_typeContext ctx) { }
- @Override
- public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
- @Override
- public void enterMultDivExpression(MultDivExpressionContext ctx) { }
- @Override
- public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
- @Override
- public void enterParForStatement(ParForStatementContext ctx) { }
- @Override
- public void enterPathStatement(PathStatementContext ctx) { }
- @Override
- public void enterPowerExpression(PowerExpressionContext ctx) { }
- @Override
- public void enterRelationalExpression(RelationalExpressionContext ctx) { }
- @Override
- public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
- @Override
- public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
- @Override
- public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterUnaryExpression(UnaryExpressionContext ctx) { }
- @Override
- public void enterWhileStatement(WhileStatementContext ctx) { }
-
- @Override
- public void visitErrorNode(ErrorNode arg0) { }
- @Override
- public void visitTerminal(TerminalNode arg0) { }
- @Override
- public void exitEveryRule(ParserRuleContext arg0) {}
- // --------------------------------------------------------------------
- private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
- // expr.setFilename(helper.getCurrentFileName());
- String txt = ctx.getText();
- expr.setFilename(_currentPath);
- expr.setBeginLine(ctx.start.getLine());
- expr.setBeginColumn(ctx.start.getCharPositionInLine());
- expr.setEndLine(ctx.stop.getLine());
- expr.setEndColumn(ctx.stop.getCharPositionInLine());
- if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
- expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
- }
- }
-
- private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
- String txt = ctx.getText();
- stmt.setFilename(helper.getCurrentFileName());
- stmt.setBeginLine(ctx.start.getLine());
- stmt.setBeginColumn(ctx.start.getCharPositionInLine());
- stmt.setEndLine(ctx.stop.getLine());
- stmt.setEndColumn(ctx.stop.getCharPositionInLine());
- if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
- stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
- }
- }
-
- // For now do no type checking, let validation handle it.
- // This way parser doesn't have to open metadata file
- @Override
- public void exitAddSubExpression(AddSubExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- // Addition and subtraction operator same as DML
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
-
-
- @Override
- public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("//") == 0) {
- dmlOperator = "%/%";
- }
- else if(ctx.op.getText().compareTo("%") == 0) {
- dmlOperator = "%%";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected // or %)", ctx.op);
- return;
- }
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitUnaryExpression(UnaryExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- if(ctx.left.info.expr instanceof IntIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else if(ctx.left.info.expr instanceof DoubleIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else {
- Expression right = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.op.getText().compareTo("-") == 0) {
- right = new IntIdentifier(-1, fileName, line, col, line, col);
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp("*");
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(right);
- }
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitMultDivExpression(MultDivExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitPowerExpression(PowerExpressionContext ctx) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("**") == 0) {
- dmlOperator = "^";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected **)", ctx.op);
- return;
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // TODO:
-// @Override
-// public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
-// Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
-// ctx.info.expr = new BinaryExpression(bop);
-// ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
-// ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitRelationalExpression(RelationalExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
-// String fileName = helper.getCurrentFileName();
-// int line = ctx.start.getLine();
-// int col = ctx.start.getCharPositionInLine();
-// ArrayList<ParameterExpression> paramExpression = new ArrayList<ParameterExpression>();
-// paramExpression.add(new ParameterExpression(null, ctx.left.info.expr));
-// paramExpression.add(new ParameterExpression(null, ctx.right.info.expr));
-// ParameterExpression operator = new ParameterExpression(null, new StringIdentifier(ctx.op.getText(), fileName, line, col, line, col));
-// paramExpression.add(operator);
-//
-// try {
-// BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression("ppred", paramExpression, fileName, line, col, line, col);
-// if (bife != null){
-// // It is a builtin function
-// ctx.info.expr = bife;
-// return;
-// }
-// }
-// catch(Exception e) {}
-// helper.notifyErrorListeners("Cannot parse relational expression", ctx.getStart());
-
- Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
- ctx.info.expr = new RelationalExpression(rop);
- ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("&") == 0 || ctx.op.getText().compareTo("and") == 0) {
- dmlOperator = "&";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected &)", ctx.op);
- return;
- }
-
- Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("|") == 0 || ctx.op.getText().compareTo("or") == 0) {
- dmlOperator = "|";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected |)", ctx.op);
- return;
- }
-
- Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitAtomicExpression(AtomicExpressionContext ctx) {
- ctx.info.expr = ctx.left.info.expr;
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-// @Override
-// public void exitConstBooleanIdExpression(ConstBooleanIdExpressionContext ctx) {
-// boolean val = false;
-// if(ctx.getText().compareTo("TRUE") == 0) {
-// val = true;
-// }
-// else if(ctx.getText().compareTo("FALSE") == 0) {
-// val = false;
-// }
-// else {
-// helper.notifyErrorListeners("cannot parse the boolean value: \'" + ctx.getText() + "\'", ctx.getStart());
-// return;
-// }
-// int linePosition = ctx.start.getLine();
-// int charPosition = ctx.start.getCharPositionInLine();
-// ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
- @Override
- public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
- try {
- double val = Double.parseDouble(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the double value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
- try {
- long val = Long.parseLong(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the integer value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
- String val = "";
- String text = ctx.getText();
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
- return;
- }
-
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitDataIdExpression(DataIdExpressionContext ctx) {
- ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
- setFileLineColumn(ctx.info.expr, ctx);
-// if(ctx.getChild(0) instanceof DataIdentifierContext) {
-// ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
-// }
-// else {
-// String msg = "cannot evaluate data expression ... strange";
-// helper.notifyErrorListeners(msg, ctx.start);
-// }
- }
-
- @Override
- public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
- // This is either a function, or variable with namespace
- // By default, it assigns to a data type
- ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- }
-
-
- private Expression incrementByOne(Expression expr, ParserRuleContext ctx) {
- // For maintaining semantic consistency, we have decided to keep 1-based indexing
- // If in future, PyDML becomes more popular than DML, this can be switched.
- return expr;
- }
-
- @Override
- public void exitIndexedExpression(IndexedExpressionContext ctx) {
- ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- try {
- ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
-
- ArrayList<Expression> rowIndices = new ArrayList<Expression>();
- ArrayList<Expression> colIndices = new ArrayList<Expression>();
-
- boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
- boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
- boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
- boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
-
- if(!isRowLower && !isRowUpper) {
- // both not set
- rowIndices.add(null); rowIndices.add(null);
- }
- else if(isRowLower && isRowUpper) {
- // both set
- rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
- rowIndices.add(ctx.rowUpper.info.expr);
- }
- else if(isRowLower && !isRowUpper) {
- // only row set
- rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
- return;
- }
-
- if(!isColLower && !isColUpper) {
- // both not set
- colIndices.add(null); colIndices.add(null);
- }
- else if(isColLower && isColUpper) {
- colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
- colIndices.add(ctx.colUpper.info.expr);
- }
- else if(isColLower && !isColUpper) {
- colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
- return;
- }
- exprList.add(rowIndices);
- exprList.add(colIndices);
- ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot set the indices", ctx.start);
- return;
- }
- }
-
- private ConstIdentifier getConstIdFromString(String varValue, Token start) {
- // Both varName and varValue are correct
- int linePosition = start.getLine();
- int charPosition = start.getCharPositionInLine();
- try {
- long val = Long.parseLong(varValue);
- return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e) {
- try {
- double val = Double.parseDouble(varValue);
- return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e1) {
- try {
- if(varValue.compareTo("True") == 0 || varValue.compareTo("False") == 0) {
- boolean val = false;
- if(varValue.compareTo("True") == 0) {
- val = true;
- }
- return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- else {
- String val = "";
- String text = varValue;
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- val = text;
- // the commandline parameters can be passed without any quotes
-// helper.notifyErrorListeners("something wrong while parsing string ... strange", start);
-// return null;
- }
- return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- }
- catch(Exception e3) {
- helper.notifyErrorListeners("unable to cast the commandline parameter into int/double/boolean/string", start);
- return null;
- }
- }
- }
- }
-
- private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
-
- if(!varName.startsWith("$")) {
- helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
- return;
- }
-
- String varValue = null;
- for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
- if(arg.getKey().trim().compareTo(varName) == 0) {
- if(varValue != null) {
- helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
- return;
- }
- else {
- varValue = arg.getValue().trim();
- }
- }
- }
-
- if(varValue == null) {
- // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
- return;
- }
-
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- if(varValue.trim().compareTo("") == 0)
- return;
-
- dataInfo.expr = getConstIdFromString(varValue, start);
- }
-
- @Override
- public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
- }
-
- @Override
- public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
- }
-
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitImportStatement(ImportStatementContext ctx)
- {
- //prepare import filepath
- String filePath = ctx.filePath.getText();
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
- namespace = ctx.namespace.getText();
- }
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- //concatenate working directory to filepath
- filePath = _workingDir + File.separator + filePath;
-
- DMLProgram prog = null;
- try {
- prog = (new PyDMLParserWrapper()).doParse(filePath, null, argVals);
- } catch (ParseException e) {
- helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
- return;
- }
- // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
- if(prog == null) {
- helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
- return;
- }
- else {
- ctx.info.namespaces = new HashMap<String, DMLProgram>();
- ctx.info.namespaces.put(namespace, prog);
- ctx.info.stmt = new ImportStatement();
- ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
- ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
- ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
- }
- }
-
- @Override
- public void exitAssignmentStatement(AssignmentStatementContext ctx) {
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = ctx.source.info.expr;
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
- return;
- }
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
-
- private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
- try {
- ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid function call", ctx.start);
- return;
- }
- }
-
- private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
- return;
- }
- Expression expr = paramExpression.get(0).getExpr();
- if(expr == null) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- try {
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- }
-
- private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() < 2){
- helper.notifyErrorListeners("incorrect usage of load function (atleast 2 arguments required)", ctx.start);
- return;
- }
- if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
- // && paramExpression.get(0).getName() == null
- // correct usage of identifier
- // if(paramExpression.get(1).getName() == null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- HashMap<String, Expression> varParams = new HashMap<String, Expression>();
- varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
- for(int i = 2; i < paramExpression.size(); i++) {
- // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
- varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
- }
-
- DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
- ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
- return;
- //}
- }
-
- helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
- return;
-
- }
-
- private boolean inDefaultNamespace(String namespace) {
- return namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0;
- }
-
- // Returns 0, 1 or -1 (for error)
- private int getAxis(ParameterizedExpressionContext ctx) {
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- if(ctx.paramName.getText().compareTo("axis") != 0) {
- return -1;
- }
- }
-
- String val = ctx.paramVal.getText();
- if(val != null && val.compareTo("0") == 0) {
- return 0;
- }
- else if(val != null && val.compareTo("1") == 0) {
- return 1;
- }
-
- return -1;
- }
-
- private String getPythonAggFunctionNames(String functionName, int axis) {
- if(axis != 0 && axis != 1) {
- return functionName;
- }
- // axis=0 maps to column-wise computation and axis=1 maps to row-wise computation
-
- if(functionName.compareTo("sum") == 0) {
- return axis == 0 ? "colSums" : "rowSums";
- }
- else if(functionName.compareTo("mean") == 0) {
- return axis == 0 ? "colMeans" : "rowMeans";
- }
- else if(functionName.compareTo("avg") == 0) {
- return axis == 0 ? "colMeans" : "rowMeans";
- }
- else if(functionName.compareTo("max") == 0) {
- return axis == 0 ? "colMaxs" : "rowMaxs";
- }
- else if(functionName.compareTo("min") == 0) {
- return axis == 0 ? "colMins" : "rowMins";
- }
- else if(functionName.compareTo("argmin") == 0) {
- return axis == 0 ? "Not Supported" : "rowIndexMin";
- }
- else if(functionName.compareTo("argmax") == 0) {
- return axis == 0 ? "Not Supported" : "rowIndexMax";
- }
- else if(functionName.compareTo("cumsum") == 0) {
- return axis == 0 ? "cumsum" : "Not Supported";
- }
- else if(functionName.compareTo("transpose") == 0) {
- return axis == 0 ? "Not Supported" : "Not Supported";
- }
- else if(functionName.compareTo("trace") == 0) {
- return axis == 0 ? "Not Supported" : "Not Supported";
- }
- else {
- return functionName;
- }
- }
-
- private ConvertedDMLSyntax convertPythonBuiltinFunctionToDMLSyntax(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression,
- List<ParameterizedExpressionContext> paramCtx, Token fnName, String fileName, int line, int col) {
- // ===========================================================================================
- // Check function name, namespace, parameters (#params & possible values) and throw useful hints
- if(inDefaultNamespace(namespace) && functionName.compareTo("len") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 1 arguments", fnName);
- return null;
- }
- functionName = "length";
- }
- else if(functionName.compareTo("sum") == 0 || functionName.compareTo("mean") == 0 || functionName.compareTo("avg") == 0 ||
- functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 ||
- functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 ||
- functionName.compareTo("cumsum") == 0 || functionName.compareTo("transpose") == 0 || functionName.compareTo("trace") == 0) {
- // 0 maps row-wise computation and 1 maps to column-wise computation
-
- // can mean sum of all cells or row-wise or columnwise sum
- if(inDefaultNamespace(namespace) && paramExpression.size() == 1) {
- // sum(x) => sum(x)
- // otherwise same function name
- if(functionName.compareTo("avg") == 0) {
- functionName = "mean";
- }
- else if(functionName.compareTo("transpose") == 0) {
- functionName = "t";
- }
- else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
- return null;
- }
- }
- else if(!inDefaultNamespace(namespace) && paramExpression.size() == 0) {
- // x.sum() => sum(x)
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- // otherwise same function name
- if(functionName.compareTo("avg") == 0) {
- functionName = "mean";
- }
- else if(functionName.compareTo("transpose") == 0) {
- functionName = "t";
- }
- else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
- return null;
- }
- }
- else if(inDefaultNamespace(namespace) && paramExpression.size() == 2) {
- // sum(x, axis=1) => rowSums(x)
- int axis = getAxis(paramCtx.get(1));
- if(axis == -1 && (functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 )) {
- // Do nothing
- // min(2, 3)
- }
- else if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- else {
- ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
- temp.add(paramExpression.get(0));
- paramExpression = temp;
- functionName = getPythonAggFunctionNames(functionName, axis);
- if(functionName.compareTo("Not Supported") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- }
- }
- else if(!inDefaultNamespace(namespace) && paramExpression.size() == 1) {
- // x.sum(axis=1) => rowSums(x)
- int axis = getAxis(paramCtx.get(0));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- else {
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- functionName = getPythonAggFunctionNames(functionName, axis);
- if(functionName.compareTo("Not Supported") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- }
- }
- else {
- helper.notifyErrorListeners("Incorrect number of arguments for the builtin function \'" + functionName + "\'.", fnName);
- return null;
- }
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("concatenate") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments (Note: concatenate append columns of two matrices)", fnName);
- return null;
- }
- functionName = "append";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("minimum") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
- return null;
- }
- functionName = "min";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("maximum") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
- return null;
- }
- functionName = "max";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(!inDefaultNamespace(namespace) && functionName.compareTo("shape") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
- return null;
- }
-
- int axis = getAxis(paramCtx.get(0));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
- return null;
- }
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(axis == 0) {
- functionName = "nrow";
- }
- else if(axis == 1) {
- functionName = "ncol";
- }
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("random.normal") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (number of rows, number of columns, sparsity)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
- paramExpression.get(2).setName("sparsity");
- paramExpression.add(new com.ibm.bi.dml.parser.ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col)));
- functionName = "rand";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("random.uniform") == 0) {
- if(paramExpression.size() != 5) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 5 arguments (number of rows, number of columns, sparsity, min, max)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
- paramExpression.get(2).setName("sparsity");
- paramExpression.get(3).setName("min");
- paramExpression.get(4).setName("max");
- paramExpression.add(new com.ibm.bi.dml.parser.ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col)));
- functionName = "rand";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("full") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (constant double value, number of rows, number of columns)", fnName);
- return null;
- }
- paramExpression.get(1).setName("rows");
- paramExpression.get(2).setName("cols");
- functionName = "matrix";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("matrix") == 0) {
- // This can either be string initializer or as.matrix function
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument (either string or double value)", fnName);
- return null;
- }
-
- if(paramExpression.get(0).getExpr() instanceof StringIdentifier) {
- String initializerString = ((StringIdentifier)paramExpression.get(0).getExpr()).getValue().trim();
- if(!initializerString.startsWith("[") || !initializerString.endsWith("]")) {
- helper.notifyErrorListeners("Incorrect initializer string for builtin function \'" + functionName + "\' (Eg: matrix(\"[1 2 3; 4 5 6]\"))", fnName);
- return null;
- }
- int rows = StringUtils.countMatches(initializerString, ";") + 1;
-
- // Make sure user doesnot have pretty string
- initializerString = initializerString.replaceAll("; ", ";");
- initializerString = initializerString.replaceAll(" ;", ";");
- initializerString = initializerString.replaceAll("\\[ ", "\\[");
- initializerString = initializerString.replaceAll(" \\]", "\\]");
-
- // Each row has ncol-1 spaces
- // #spaces = nrow * (ncol-1)
- // ncol = (#spaces / nrow) + 1
- int cols = (StringUtils.countMatches(initializerString, " ") / rows) + 1;
-
- initializerString = initializerString.replaceAll(";", " ");
- initializerString = initializerString.replaceAll("\\[", "");
- initializerString = initializerString.replaceAll("\\]", "");
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new StringIdentifier(initializerString, fileName, line, col, line, col)));
- paramExpression.add(new ParameterExpression("rows", new IntIdentifier(rows, fileName, line, col, line, col)));
- paramExpression.add(new ParameterExpression("cols", new IntIdentifier(cols, fileName, line, col, line, col)));
- }
- else {
- functionName = "as.matrix";
- }
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("scalar") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.scalar";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("float") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.double";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("int") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.integer";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("bool") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.logical";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(!inDefaultNamespace(namespace) && functionName.compareTo("reshape") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (number of rows, number of columns)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
-
- ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
- temp.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- temp.add(paramExpression.get(0));
- temp.add(paramExpression.get(1));
- paramExpression = temp;
-
- functionName = "matrix";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("removeEmpty") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
- return null;
- }
- int axis = getAxis(paramCtx.get(1));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
- return null;
- }
- StringIdentifier marginVal = null;
- if(axis == 0) {
- marginVal = new StringIdentifier("rows", fileName, line, col, line, col);
- }
- else {
- marginVal = new StringIdentifier("cols", fileName, line, col, line, col);
- }
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("margin");
- paramExpression.get(1).setExpr(marginVal);
- functionName = "removeEmpty";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("replace") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
- return null;
- }
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("pattern");
- paramExpression.get(2).setName("replacement");
- functionName = "replace";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("range") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
- return null;
- }
- functionName = "seq";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("norm.cdf") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, mean, sd)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("mean");
- paramExpression.get(2).setName("sd");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("normal", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("expon.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, mean)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("mean");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("exp", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("chi.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("chisq", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("f.cdf") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, df1, df2)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df1");
- paramExpression.get(2).setName("df2");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("f", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("t.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("t", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("percentile") == 0) {
- if(paramExpression.size() != 2 && paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts either 2 or 3 arguments", fnName);
- return null;
- }
- functionName = "quantile";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arcsin") == 0) {
- functionName = "asin";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arccos") == 0) {
- functionName = "acos";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arctan") == 0) {
- functionName = "atan";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("load") == 0) {
- functionName = "read";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("eigen") == 0) {
- functionName = "eig";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("power") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
- return null;
- }
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
- return null;
- }
- }
-
- ConvertedDMLSyntax retVal = new ConvertedDMLSyntax();
- retVal.namespace = namespace;
- retVal.functionName = functionName;
- retVal.paramExpression = paramExpression;
- return retVal;
- }
-
- class ConvertedDMLSyntax {
- public String namespace;
- public String functionName;
- public ArrayList<ParameterExpression> paramExpression;
- };
-
- private Expression getOperatorExpression(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression) {
- String dmlOperator = null;
-
- if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
- if(paramExpression.size() == 2) {
- dmlOperator = "%*%";
- }
- }
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("power") == 0) {
-// if(paramExpression.size() == 2) {
-// dmlOperator = "^";
-// }
-// }
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("multiply") == 0) {
-// if(paramExpression.size() == 2) {
-// dmlOperator = "*";
-// }
-// }
-
- if(dmlOperator != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- Expression expr = new BinaryExpression(bop);
- ((BinaryExpression)expr).setLeft(paramExpression.get(0).getExpr());
- ((BinaryExpression)expr).setRight(paramExpression.get(1).getExpr());
- return expr;
- }
-
- return null;
- }
-
- @Override
- public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setPrintStatement(ctx, functionName);
- return;
- }
- else if(functionName.compareTo("save") == 0 && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setOutputStatement(ctx);
- return;
- }
-
- boolean ignoreLValue = false;
- if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
- helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
- return;
- }
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- }
-
- // Because the user might want to define a corresponding UDF function
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("seq") == 0) {
-// helper.notifyErrorListeners("No function with name \'" + functionName + "\' exists for PyDML. Use \'range\' instead.", ctx.name);
-// return;
-// }
- // ===========================================================================================
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(!ignoreLValue) {
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
- if(operatorExpr != null) {
- setFileLineColumn(operatorExpr, ctx);
- setAssignmentStatement(target, operatorExpr, ctx);
- return;
- }
-
- if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
- if(!helper.validateBuiltinFunctions(ctx)) {
- return; // it is a built-in function and validation failed, so donot proceed ahead.
- }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setAssignmentStatement(target, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setAssignmentStatement(target, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setAssignmentStatement(target, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- }
-
- setAssignmentStatement(target, functCall, ctx);
- }
-
-
- @Override
- public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- // System.out.println(ctx.name.getText() + ">>" + namespace + " " + functionName);
- }
-
- Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
- if(operatorExpr != null) {
- ctx.info.expr = operatorExpr;
- setFileLineColumn(operatorExpr, ctx);
- return;
- }
-
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- ctx.info.expr = bife;
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- ctx.info.expr = pbife;
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (dbife != null){
- ctx.info.expr = dbife;
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
- }
-
- private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
- ctx.info.stmt = new MultiAssignmentStatement(target, expression);
- ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitFunctionCallMultiAssignmentStatement(
- FunctionCallMultiAssignmentStatementContext ctx) {
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- }
-
- // No need to support dot() function since it will never return multi-assignment function
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
- for(DataIdentifierContext dataCtx : ctx.targetList) {
- if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
- targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setMultiAssignmentStatement(targetList, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setMultiAssignmentStatement(targetList, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setMultiAssignmentStatement(targetList, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return;
- }
- }
-
- setMultiAssignmentStatement(targetList, functCall, ctx);
- }
-
- private StatementBlock getStatementBlock(Statement current) {
- return PyDMLParserWrapper.getStatementBlock(current);
- }
-
- @Override
- public void exitIfStatement(IfStatementContext ctx) {
- IfStatement ifStmt = new IfStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- ifStmt.setConditionalPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ifStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.ifBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.ifBody) {
- ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksIfBody();
- }
-
- if(ctx.elseBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.elseBody) {
- ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksElseBody();
- }
-
- ctx.info.stmt = ifStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitWhileStatement(WhileStatementContext ctx) {
- WhileStatement whileStmt = new WhileStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- whileStmt.setPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- whileStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- whileStmt.mergeStatementBlocks();
- }
-
- ctx.info.stmt = whileStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitForStatement(ForStatementContext ctx) {
- ForStatement forStmt = new ForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = null;
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.iterPred.info.increment != null) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- forStmt.setPredicate(predicate);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- forStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = forStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitParForStatement(ParForStatementContext ctx) {
- ParForStatement parForStmt = new ParForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = new HashMap<String, String>();
- if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
- for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
- parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
- }
- }
-
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
-
- if( ctx.iterPred.info.increment != null ) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- parForStmt.setPredicate(predicate);
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- parForStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = parForStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
-
-
-
- // ----------------------------------------------------------------------
- @Override
- public void exitMl_type(Ml_typeContext ctx) { }
-
- private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
- ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
- for(TypedArgNoAssignContext paramCtx : ctx) {
- DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
- String dataType = null;
- String valueType = null;
-
- if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
- || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
- dataType = "scalar";
- }
- else {
- dataType = paramCtx.paramType.dataType().getText();
- }
-
- if(dataType.compareTo("matrix") == 0) {
- // matrix
- dataId.setDataType(DataType.MATRIX);
- }
- else if(dataType.compareTo("scalar") == 0) {
- // scalar
- dataId.setDataType(DataType.SCALAR);
- }
- else {
- helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
- return null;
- }
-
- valueType = paramCtx.paramType.valueType().getText();
- if(valueType.compareTo("int") == 0) {
- dataId.setValueType(ValueType.INT);
- }
- else if(valueType.compareTo("str") == 0) {
- dataId.setValueType(ValueType.STRING);
- }
- else if(valueType.compareTo("bool") == 0) {
- dataId.setValueType(ValueType.BOOLEAN);
- }
- else if(valueType.compareTo("float") == 0) {
- dataId.setValueType(ValueType.DOUBLE);
- }
- else {
- helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
- return null;
- }
- retVal.add(dataId);
- }
- return retVal;
- }
-
- @Override
- public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
- FunctionStatement functionStmt = new FunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
-
- if(ctx.body.size() > 0) {
- // handle function body
- // Create arraylist of one statement block
- ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
- for(StatementContext stmtCtx : ctx.body) {
- body.add(getStatementBlock(stmtCtx.info.stmt));
- }
- functionStmt.setBody(body);
- functionStmt.mergeStatementBlocks();
- }
- else {
- helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
- return;
- }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
- @Override
- public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
- ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
- // set other parameters
- HashMap<String, String> otherParams = new HashMap<String,String>();
- boolean atleastOneClassName = false;
- for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
- String paramName = otherParamCtx.paramName.getText();
- String val = "";
- String text = otherParamCtx.paramVal.getText();
- // First unquote the string
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- // Empty value allowed
- }
- else {
- helper.notifyErrorListeners("the value of user parameter for external function should be of type string", ctx.start);
- return;
- }
- otherParams.put(paramName, val);
- if(paramName.compareTo("classname") == 0) {
- atleastOneClassName = true;
- }
- }
- functionStmt.setOtherParams(otherParams);
- if(!atleastOneClassName) {
- helper.notifyErrorListeners("the parameter \'className\' needs to be passed for defExternal", ctx.start);
- return;
- }
-
-// if(ctx.body.size() > 0) {
-// // handle function body
-// // Create arraylist of one statement block
-// ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
-// for(StatementContext stmtCtx : ctx.body) {
-// body.add(getStatementBlock(stmtCtx.info.stmt));
-// }
-// ((ExternalFunctionStatement) functionStmt).setBody(body);
-// ((ExternalFunctionStatement) functionStmt).mergeStatementBlocks();
-// }
-// else {
-// helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
-// return;
-// }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
-
- @Override
- public void exitPathStatement(PathStatementContext ctx) {
- PathStatement stmt = new PathStatement(ctx.pathValue.getText());
- String filePath = ctx.pathValue.getText();
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- _workingDir = filePath;
- ctx.info.stmt = stmt;
- }
-
- @Override
- public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
- if(!ctx.commandLineParam.getText().startsWith("$")) {
- helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
- return;
- }
-
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = null;
- if(ctx.commandLineParam.dataInfo.expr != null) {
- // Since commandline parameter is set
- // The check of following is done in fillExpressionInfoCommandLineParameters:
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- source = ctx.commandLineParam.dataInfo.expr;
- }
- else {
- source = ctx.source.info.expr;
- }
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
- return;
- }
-
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
- // ----------------------------------------------------------------------
- @Override
- public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
-
-
- @Override
- public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
-
- @Override
- public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
- @Override
- public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
- @Override
- public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("matrix") == 0
- || ctx.ID().getText().compareTo("scalar") == 0
- ) {
- // Do nothing
- }
- else if(ctx.ID().getText().compareTo("Matrix") == 0)
- helper.notifyErrorListeners("incorrect datatype (Hint: use matrix instead of Matrix)", ctx.start);
- else if(ctx.ID().getText().compareTo("Scalar") == 0)
- helper.notifyErrorListeners("incorrect datatype (Hint: use scalar instead of Scalar)", ctx.start);
- else if( ctx.ID().getText().compareTo("int") == 0
- || ctx.ID().getText().compareTo("str") == 0
- || ctx.ID().getText().compareTo("bool") == 0
- || ctx.ID().getText().compareTo("float") == 0
- ) {
- helper.notifyErrorListeners("expected datatype but found a valuetype (Hint: use matrix or scalar instead of " + ctx.ID().getText() + ")", ctx.start);
- }
- else {
- helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
- }
- }
-
- @Override
- public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
- @Override
- public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
- @Override
- public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
- @Override
- public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
- @Override
- public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
-
- @Override
- public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = null;
- }
-
- @Override
- public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
- if(ctx.ID().getText().compareTo("range") != 0) {
- helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'range\'", ctx.start);
- return;
- }
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = ctx.increment.info.expr;
- }
-
- @Override
- public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
- @Override
- public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
-
- @Override
- public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
- boolean val = false;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
- @Override
- public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
- boolean val = true;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitIgnoreNewLine(IgnoreNewLineContext ctx) {
- // Introduce empty StatementInfo
- // This is later ignored by PyDMLParserWrapper
- try {
- ctx.info.stmt = new AssignmentStatement(null, null, 0, 0, 0, 0);
- ctx.info.stmt.setEmptyNewLineStatement(true);
- } catch (LanguageException e) {
- e.printStackTrace();
- }
-
- }
- @Override
- public void enterValueDataTypeCheck(ValueDataTypeCheckContext ctx) { }
- @Override
- public void exitValueDataTypeCheck(ValueDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("int") == 0
- || ctx.ID().getText().compareTo("str") == 0
- || ctx.ID().getText().compareTo("bool") == 0
- || ctx.ID().getText().compareTo("float") == 0
- ) {
- // Do nothing
- }
- else if(ctx.ID().getText().compareTo("integer") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use int instead of integer)", ctx.start);
- else if(ctx.ID().getText().compareTo("double") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use float instead of double)", ctx.start);
- else if(ctx.ID().getText().compareTo("boolean") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use bool instead of boolean)", ctx.start);
- else if(ctx.ID().getText().compareTo("string") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use str instead of string)", ctx.start);
- else {
- helper.notifyErrorListeners("incorrect valuetype (expected int, str, bool or float)", ctx.start);
- }
- }
-
-
-}
[71/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
index 2ee9680..3f8ba1b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.HashMap;
@@ -35,28 +35,28 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
/**
* MR job class for submitting parfor remote MR jobs, controlling its execution and obtaining results.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
index f92b50a..997cdda 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.List;
@@ -27,21 +27,21 @@ import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.utils.Statistics;
/**
* TODO robustness on failures (cleanup files)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
index 1338acb..01d3f09 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,20 +26,20 @@ import org.apache.spark.Accumulator;
import org.apache.spark.TaskContext;
import org.apache.spark.api.java.function.PairFlatMapFunction;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.Task.TaskType;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.LocalFileUtils;
import scala.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
index 6aec002..d982193 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.HashMap;
@@ -28,25 +28,25 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.PairWritableCell;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.Task.TaskType;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
index 580a4e3..4d6dd5c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.HashMap;
@@ -34,9 +34,9 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.NLineInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.controlprogram.parfor.Task.TaskType;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
index 8243b6d..3960b05 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedNLineInputFormat.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
@@ -24,8 +24,8 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.NLineInputFormat;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
/**
* Specific extension of NLineInputFormat in order to ensure data colocation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
index e052f41..bef9069 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
/**
* Wrapper for job return of ParFor REMOTE for transferring statistics and result symbol table.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
index b74ae1f..a5587c4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.HashMap;
@@ -36,24 +36,24 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.lib.NLineInputFormat;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
/**
* MR job class for submitting parfor remote MR jobs, controlling its execution and obtaining results.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
index 255ad15..f65c755 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.List;
@@ -26,13 +26,13 @@ import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.utils.Statistics;
/**
* This class serves two purposes: (1) isolating Spark imports to enable running in
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
index bfa7e96..9186876 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,11 +24,11 @@ import org.apache.spark.Accumulator;
import org.apache.spark.TaskContext;
import org.apache.spark.api.java.function.PairFlatMapFunction;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.util.LocalFileUtils;
import scala.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
index 78c884b..92f1959 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.ArrayList;
@@ -32,20 +32,20 @@ import org.apache.hadoop.mapred.Reporter;
import scala.Tuple2;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.utils.Statistics;
/**
* Common functionalities for parfor workers in MR jobs. Used by worker wrappers in
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
index 9d6f320..dcf227b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.HashMap;
@@ -28,20 +28,20 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.StatisticMonitor;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.utils.Statistics;
/**
* Remote ParWorker implementation, realized as MR mapper.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
index c4ff3c5..d20b46f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* Due to independence of all iterations, any result has the following properties:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
index 745ae08..6934ab6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
index 8020719..621853a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.BufferedWriter;
import java.io.File;
@@ -38,29 +38,29 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.Cell;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.util.Cell;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
index 77b3bb5..cd7fc18 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* Local in-memory realization of result merge. If the resulting matrix is
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
index 041b0bd..3bfee4b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.hadoop.io.WritableComparable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
index 0a406e3..e952556 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.ArrayList;
import java.util.Collection;
@@ -30,26 +30,26 @@ import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixCell;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Statistics;
/**
* MR job class for submitting parfor result merge MR jobs.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
index 52690ee..e8cb85b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
@@ -25,15 +25,15 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixCell;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* Remote resultmerge mapper implementation that does the preprocessing
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
index d93949f..9c0cb76 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class ResultMergeRemotePartitioning implements Partitioner<ResultMergeTaggedMatrixIndexes, TaggedMatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
index 5066a82..0cf1642 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.Collection;
@@ -30,16 +30,16 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixCell;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* Remote result merge reducer that receives all worker results partitioned by
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
index 4f81764..67f9e6b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
index 6765bc3..9eeb84a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.utils.Statistics;
/**
* MR job class for submitting parfor result merge MR jobs.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
index 7df2cd1..3f5a834 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.Iterator;
@@ -23,11 +23,11 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.DataConverter;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
index af87aa3..86b66bc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.DataInput;
import java.io.DataOutput;
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* This class serves as composite key for the remote result merge job
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
index c2ff9ac..9a99101 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.Serializable;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* A task is a logical group of one or multiple iterations (each iteration is assigned to exactly one task).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitioner.java
index 0b7eb4e..da60c5c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitioner.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.List;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* This is the base class for all task partitioner. For this purpose it stores relevant information such as
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
index cd03544..adc1941 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.LinkedList;
import java.util.List;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.Task.TaskType;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* This factoring task partitioner virtually iterates over the given FOR loop (from, to, incr),
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
index 914f387..5772a9c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* Factoring with maximum constraint (e.g., if LIX matrix out-of-core and we need
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
index 0d1c022..a014c67 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* Factoring with minimum constraint (e.g., if communication is expensive)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
index 54fc72a..9456e57 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.LinkedList;
import java.util.List;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.Task.TaskType;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* This naive task partitioner virtually iterates over the given FOR loop (from, to, incr),
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerNaive.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
index 2ad4806..3a68f1f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* This static task partitioner virtually iterates over the given FOR loop (from, to, incr),
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerStatic.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
index 20be635..03523d9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
+package org.apache.sysml.runtime.controlprogram.parfor;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
/**
* This static task partitioner virtually iterates over the given FOR loop (from, to, incr),
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
index 7b3423a..53d92cb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.HashMap;
import java.util.LinkedList;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MetaData;
/**
* Merged MR Job instruction to hold the actually merged instruction as well as offsets of
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
index f76f515..3680002 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.HashMap;
import java.util.LinkedList;
@@ -23,10 +23,10 @@ import java.util.LinkedList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.data.Pair;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
index d385fc7..85f9fb5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.LinkedList;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.utils.Statistics;
public class PiggybackingWorkerTimeSequential extends PiggybackingWorker
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
index cad9967..908f133 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.LinkedList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
index 9c89532..17608f4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.LinkedList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.utils.Statistics;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
index 1c5024c..f2a0323 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.util.HashMap;
import java.util.LinkedList;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.data.Pair;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybackingUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybackingUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybackingUtils.java
index ede22dc..c637f15 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybackingUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybackingUtils.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
+package org.apache.sysml.runtime.controlprogram.parfor.mqo;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
public class RuntimePiggybackingUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
index 3e1f07f..0bc2931 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
/**
* Base class for all potential cost estimators
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
index d4a8716..19f78e4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.Optimizer.CostModelType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.CostModelType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
index c78777a..92ca2ca 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
/**
* Cost Estimator for runtime programs. It uses a offline created performance profile
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
index 747c757..3a1c143 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
index f083d49..39edb19 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
index 0c09240..97a533d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
@@ -16,9 +16,9 @@
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
index 1fdcbde..23b52d8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
import java.util.Collection;
@@ -23,11 +23,11 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.Set;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.Lop;
+import org.apache.sysml.lops.LopProperties;
+import org.apache.sysml.lops.Lop;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
/**
* Internal representation of a plan alternative for program blocks and instructions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNodeStatistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNodeStatistics.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNodeStatistics.java
index 6cb2d31..a0ce00d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNodeStatistics.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNodeStatistics.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.DataFormat;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTree.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTree.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTree.java
index 7354fce..b96e5a8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTree.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTree.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.controlprogram.parfor.opt;
+package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.Optimizer.PlanInputType;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.PlanInputType;
/**
* Represents a complete plan of a top-level parfor. This includes the internal
[38/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/TernaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/TernaryOp.java b/src/main/java/com/ibm/bi/dml/hops/TernaryOp.java
deleted file mode 100644
index c1cb96b..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/TernaryOp.java
+++ /dev/null
@@ -1,1024 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.CentralMoment;
-import com.ibm.bi.dml.lops.CoVariance;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.CombineTernary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Ternary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.CombineBinary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-/* Primary use cases for now, are
- * quantile (<n-1-matrix>, <n-1-matrix>, <literal>): quantile (A, w, 0.5)
- * quantile (<n-1-matrix>, <n-1-matrix>, <scalar>): quantile (A, w, s)
- * interquantile (<n-1-matrix>, <n-1-matrix>, <scalar>): interquantile (A, w, s)
- *
- * Keep in mind, that we also have binaries for it w/o weights.
- * quantile (A, 0.5)
- * quantile (A, s)
- * interquantile (A, s)
- *
- * Note: this hop should be called AggTernaryOp in consistency with AggUnaryOp and AggBinaryOp;
- * however, since there does not exist a real TernaryOp yet - we can leave it as is for now.
- */
-
-public class TernaryOp extends Hop
-{
-
- public static boolean ALLOW_CTABLE_SEQUENCE_REWRITES = true;
-
- private OpOp3 _op = null;
-
- //ctable specific flags
- // flag to indicate the existence of additional inputs representing output dimensions
- private boolean _dimInputsPresent = false;
- private boolean _disjointInputs = false;
-
-
- private TernaryOp() {
- //default constructor for clone
- }
-
- public TernaryOp(String l, DataType dt, ValueType vt, Hop.OpOp3 o,
- Hop inp1, Hop inp2, Hop inp3) {
- super(l, dt, vt);
- _op = o;
- getInput().add(0, inp1);
- getInput().add(1, inp2);
- getInput().add(2, inp3);
- inp1.getParent().add(this);
- inp2.getParent().add(this);
- inp3.getParent().add(this);
- }
-
- // Constructor the case where TertiaryOp (table, in particular) has
- // output dimensions
- public TernaryOp(String l, DataType dt, ValueType vt, Hop.OpOp3 o,
- Hop inp1, Hop inp2, Hop inp3, Hop inp4, Hop inp5) {
- super(l, dt, vt);
- _op = o;
- getInput().add(0, inp1);
- getInput().add(1, inp2);
- getInput().add(2, inp3);
- getInput().add(3, inp4);
- getInput().add(4, inp5);
- inp1.getParent().add(this);
- inp2.getParent().add(this);
- inp3.getParent().add(this);
- inp4.getParent().add(this);
- inp5.getParent().add(this);
- _dimInputsPresent = true;
- }
-
- public OpOp3 getOp(){
- return _op;
- }
-
- public void setDisjointInputs(boolean flag){
- _disjointInputs = flag;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- try
- {
- switch( _op ) {
- case CENTRALMOMENT:
- constructLopsCentralMoment();
- break;
-
- case COVARIANCE:
- constructLopsCovariance();
- break;
-
- case QUANTILE:
- case INTERQUANTILE:
- constructLopsQuantile();
- break;
-
- case CTABLE:
- constructLopsCtable();
- break;
-
- default:
- throw new HopsException(this.printErrorLocation() + "Unknown TernaryOp (" + _op + ") while constructing Lops \n");
-
- }
- }
- catch(LopsException e) {
- throw new HopsException(this.printErrorLocation() + "error constructing Lops for TernaryOp Hop " , e);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- /**
- * Method to construct LOPs when op = CENTRAILMOMENT.
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsCentralMoment()
- throws HopsException, LopsException
- {
- if ( _op != OpOp3.CENTRALMOMENT )
- throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.CENTRALMOMENT );
-
- ExecType et = optFindExecType();
-
- if ( et == ExecType.MR )
- {
- CombineBinary combine = CombineBinary.constructCombineLop(
- OperationTypes.PreCentralMoment,
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- DataType.MATRIX, getValueType());
- combine.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- CentralMoment cm = new CentralMoment(combine, getInput()
- .get(2).constructLops(), DataType.MATRIX,
- getValueType(), et);
- cm.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
- setLineNumbers(cm);
-
- UnaryCP unary1 = new UnaryCP(cm, HopsOpOp1LopsUS
- .get(OpOp1.CAST_AS_SCALAR), getDataType(),
- getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else //CP / SPARK
- {
- CentralMoment cm = new CentralMoment(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), et);
- cm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(cm);
- setLops(cm);
- }
- }
-
- /**
- * Method to construct LOPs when op = COVARIANCE.
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsCovariance()
- throws HopsException, LopsException
- {
- if ( _op != OpOp3.COVARIANCE )
- throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.COVARIANCE );
-
- ExecType et = optFindExecType();
-
- if ( et == ExecType.MR )
- {
- // combineTertiary -> CoVariance -> CastAsScalar
- CombineTernary combine = CombineTernary
- .constructCombineLop(
- CombineTernary.OperationTypes.PreCovWeighted,
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- DataType.MATRIX, getValueType());
-
- combine.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- CoVariance cov = new CoVariance(
- combine, DataType.MATRIX, getValueType(), et);
-
- cov.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
- setLineNumbers(cov);
-
- UnaryCP unary1 = new UnaryCP(
- cov, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else //CP / SPARK
- {
- CoVariance cov = new CoVariance(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- getDataType(), getValueType(), et);
- cov.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(cov);
- setLops(cov);
- }
- }
-
- /**
- * Method to construct LOPs when op = QUANTILE | INTERQUANTILE.
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsQuantile() throws HopsException, LopsException {
-
- if ( _op != OpOp3.QUANTILE && _op != OpOp3.INTERQUANTILE )
- throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.QUANTILE + " or " + OpOp3.INTERQUANTILE );
-
- ExecType et = optFindExecType();
-
- if ( et == ExecType.MR )
- {
- CombineBinary combine = CombineBinary
- .constructCombineLop(
- OperationTypes.PreSort,
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- DataType.MATRIX, getValueType());
-
- SortKeys sort = SortKeys
- .constructSortByValueLop(
- combine,
- SortKeys.OperationTypes.WithWeights,
- DataType.MATRIX, getValueType(), et);
-
- // If only a single quantile is computed, then "pick" operation executes in CP.
- ExecType et_pick = (getInput().get(2).getDataType() == DataType.SCALAR ? ExecType.CP : ExecType.MR);
- PickByCount pick = new PickByCount(
- sort,
- getInput().get(2).constructLops(),
- getDataType(),
- getValueType(),
- (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK
- : PickByCount.OperationTypes.RANGEPICK, et_pick, false);
-
- combine.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- setOutputDimensions(pick);
- setLineNumbers(pick);
- setLops(pick);
- }
- else //CP/Spark
- {
- SortKeys sort = SortKeys.constructSortByValueLop(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- SortKeys.OperationTypes.WithWeights,
- getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
- PickByCount pick = new PickByCount(
- sort,
- getInput().get(2).constructLops(),
- getDataType(),
- getValueType(),
- (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK
- : PickByCount.OperationTypes.RANGEPICK, et, true);
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- setOutputDimensions(pick);
- setLineNumbers(pick);
- setLops(pick);
- }
- }
-
- /**
- * Method to construct LOPs when op = CTABLE.
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsCtable() throws HopsException, LopsException {
-
- if ( _op != OpOp3.CTABLE )
- throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.CTABLE );
-
- /*
- * We must handle three different cases: case1 : all three
- * inputs are vectors (e.g., F=ctable(A,B,W)) case2 : two
- * vectors and one scalar (e.g., F=ctable(A,B)) case3 : one
- * vector and two scalars (e.g., F=ctable(A))
- */
-
- // identify the particular case
-
- // F=ctable(A,B,W)
-
- DataType dt1 = getInput().get(0).getDataType();
- DataType dt2 = getInput().get(1).getDataType();
- DataType dt3 = getInput().get(2).getDataType();
- Ternary.OperationTypes tertiaryOpOrig = Ternary.findCtableOperationByInputDataTypes(dt1, dt2, dt3);
-
- // Compute lops for all inputs
- Lop[] inputLops = new Lop[getInput().size()];
- for(int i=0; i < getInput().size(); i++) {
- inputLops[i] = getInput().get(i).constructLops();
- }
-
- ExecType et = optFindExecType();
-
- //reset reblock requirement (see MR ctable / construct lops)
- setRequiresReblock( false );
-
- if ( et == ExecType.CP || et == ExecType.SPARK)
- {
- //for CP we support only ctable expand left
- Ternary.OperationTypes tertiaryOp = isSequenceRewriteApplicable(true) ? Ternary.OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT : tertiaryOpOrig;
- boolean ignoreZeros = false;
-
- if( isMatrixIgnoreZeroRewriteApplicable() ) {
- ignoreZeros = true; //table - rmempty - rshape
- inputLops[0] = ((ParameterizedBuiltinOp)getInput().get(0)).getTargetHop().getInput().get(0).constructLops();
- inputLops[1] = ((ParameterizedBuiltinOp)getInput().get(1)).getTargetHop().getInput().get(0).constructLops();
- }
-
- Ternary tertiary = new Ternary(inputLops, tertiaryOp, getDataType(), getValueType(), ignoreZeros, et);
-
- tertiary.getOutputParameters().setDimensions(_dim1, _dim2, getRowsInBlock(), getColsInBlock(), -1);
- tertiary.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- //force blocked output in CP (see below), otherwise binarycell
- if ( et == ExecType.SPARK ) {
- tertiary.getOutputParameters().setDimensions(_dim1, _dim2, -1, -1, -1);
- setRequiresReblock( true );
- }
- else
- tertiary.getOutputParameters().setDimensions(_dim1, _dim2, getRowsInBlock(), getColsInBlock(), -1);
-
- //tertiary opt, w/o reblock in CP
- setLops(tertiary);
-
-
- }
- else //MR
- {
- //for MR we support both ctable expand left and right
- Ternary.OperationTypes tertiaryOp = isSequenceRewriteApplicable() ? Ternary.OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT : tertiaryOpOrig;
-
- Group group1 = null, group2 = null, group3 = null, group4 = null;
- group1 = new Group(inputLops[0], Group.OperationTypes.Sort, getDataType(), getValueType());
- group1.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- group1.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- Ternary tertiary = null;
- // create "group" lops for MATRIX inputs
- switch (tertiaryOp)
- {
- case CTABLE_TRANSFORM:
- // F = ctable(A,B,W)
- group2 = new Group(
- inputLops[1],
- Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group2.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- group2.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- group3 = new Group(
- inputLops[2],
- Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group3.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- group3.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- if ( inputLops.length == 3 )
- tertiary = new Ternary(
- new Lop[] {group1, group2, group3},
- tertiaryOp,
- getDataType(), getValueType(), et);
- else
- // output dimensions are given
- tertiary = new Ternary(
- new Lop[] {group1, group2, group3, inputLops[3], inputLops[4]},
- tertiaryOp,
- getDataType(), getValueType(), et);
- break;
-
- case CTABLE_TRANSFORM_SCALAR_WEIGHT:
- // F = ctable(A,B) or F = ctable(A,B,1)
- group2 = new Group(
- inputLops[1],
- Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group2.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- group2.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- if ( inputLops.length == 3)
- tertiary = new Ternary(
- new Lop[] {group1,group2,inputLops[2]},
- tertiaryOp,
- getDataType(), getValueType(), et);
- else
- tertiary = new Ternary(
- new Lop[] {group1,group2,inputLops[2], inputLops[3], inputLops[4]},
- tertiaryOp,
- getDataType(), getValueType(), et);
-
- break;
-
- case CTABLE_EXPAND_SCALAR_WEIGHT:
- // F=ctable(seq(1,N),A) or F = ctable(seq,A,1)
- int left = isSequenceRewriteApplicable(true)?1:0; //left 1, right 0 (index of input data)
-
- Group group = new Group(
- getInput().get(left).constructLops(),
- Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- //TODO remove group, whenever we push it into the map task
-
- if (inputLops.length == 3)
- tertiary = new Ternary(
- new Lop[] {
- group, //matrix
- getInput().get(2).constructLops(), //weight
- new LiteralOp(left).constructLops() //left
- },
- tertiaryOp,
- getDataType(), getValueType(), et);
- else
- tertiary = new Ternary(
- new Lop[] {
- group,//getInput().get(1).constructLops(), //matrix
- getInput().get(2).constructLops(), //weight
- new LiteralOp(left).constructLops(), //left
- inputLops[3],
- inputLops[4]
- },
- tertiaryOp,
- getDataType(), getValueType(), et);
-
- break;
-
- case CTABLE_TRANSFORM_HISTOGRAM:
- // F=ctable(A,1) or F = ctable(A,1,1)
- if ( inputLops.length == 3 )
- tertiary = new Ternary(
- new Lop[] {
- group1,
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops()
- },
- tertiaryOp,
- getDataType(), getValueType(), et);
- else
- tertiary = new Ternary(
- new Lop[] {
- group1,
- getInput().get(1).constructLops(),
- getInput().get(2).constructLops(),
- inputLops[3],
- inputLops[4]
- },
- tertiaryOp,
- getDataType(), getValueType(), et);
-
- break;
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM:
- // F=ctable(A,1,W)
- group3 = new Group(
- getInput().get(2).constructLops(),
- Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group3.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- group3.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- if ( inputLops.length == 3)
- tertiary = new Ternary(
- new Lop[] {
- group1,
- getInput().get(1).constructLops(),
- group3},
- tertiaryOp,
- getDataType(), getValueType(), et);
- else
- tertiary = new Ternary(
- new Lop[] {
- group1,
- getInput().get(1).constructLops(),
- group3, inputLops[3], inputLops[4] },
- tertiaryOp,
- getDataType(), getValueType(), et);
-
- break;
-
- default:
- throw new HopsException("Invalid ternary operator type: "+_op);
- }
-
- // output dimensions are not known at compilation time
- tertiary.getOutputParameters().setDimensions(_dim1, _dim2, ( _dimInputsPresent ? getRowsInBlock() : -1), ( _dimInputsPresent ? getColsInBlock() : -1), -1);
- setLineNumbers(tertiary);
-
- Lop lctable = tertiary;
-
- if( !(_disjointInputs || tertiaryOp == Ternary.OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT) )
- {
- //no need for aggregation if (1) input indexed disjoint or one side is sequence w/ 1 increment
-
- group4 = new Group(
- tertiary, Group.OperationTypes.Sort, getDataType(),
- getValueType());
- group4.getOutputParameters().setDimensions(_dim1, _dim2, ( _dimInputsPresent ? getRowsInBlock() : -1), ( _dimInputsPresent ? getColsInBlock() : -1), -1);
- group4.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- Aggregate agg1 = new Aggregate(
- group4, HopsAgg2Lops.get(AggOp.SUM), getDataType(),
- getValueType(), ExecType.MR);
- agg1.getOutputParameters().setDimensions(_dim1, _dim2, ( _dimInputsPresent ? getRowsInBlock() : -1), ( _dimInputsPresent ? getColsInBlock() : -1), -1);
-
- agg1.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- // kahamSum is used for aggregation but inputs do not have
- // correction values
- agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
- lctable = agg1;
- }
-
- setLops( lctable );
-
- // In this case, output dimensions are known at the time of its execution, no need
- // to introduce reblock lop since table itself outputs in blocked format if dims known.
- if ( !dimsKnown() && !_dimInputsPresent ) {
- setRequiresReblock( true );
- }
- }
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += "t(" + HopsOpOp3String.get(_op) + ")";
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + _op);
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- //only quantile and ctable produce matrices
-
- switch( _op )
- {
- case CTABLE:
- // since the dimensions of both inputs must be the same, checking for one input is sufficient
- // worst case dimensions of C = [m,m]
- // worst case #nnz in C = m => sparsity = 1/m
- // for ctable_histogram also one dimension is known
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, (nnz<=dim1)?nnz:dim1);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
-
- case QUANTILE:
- // This part of the code is executed only when a vector of quantiles are computed
- // Output is a vector of length = #of quantiles to be computed, and it is likely to be dense.
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
-
- default:
- throw new RuntimeException("Memory for operation (" + _op + ") can not be estimated.");
- }
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
- if( _op == OpOp3.CTABLE ) {
- if ( _dim1 > 0 && _dim2 > 0 ) {
- // output dimensions are known, and hence a MatrixBlock is allocated
- double sp = OptimizerUtils.getSparsity(_dim1, _dim2, Math.min(nnz, _dim1));
- ret = OptimizerUtils.estimateSizeExactSparsity(_dim1, _dim2, sp );
- }
- else {
- ret = 2*4 * dim1 + //hash table (worst-case overhead 2x)
- 32 * dim1; //values: 2xint,1xObject
- }
- }
- else if ( _op == OpOp3.QUANTILE ) {
- // buffer (=2*input_size) and output (=2*input_size) for SORT operation
- // getMemEstimate works for both cases of known dims and worst-case stats
- ret = getInput().get(0).getMemEstimate() * 4;
- }
-
- return ret;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- MatrixCharacteristics[] mc = memo.getAllInputStats(getInput());
-
- switch( _op )
- {
- case CTABLE:
- boolean dimsSpec = (getInput().size() > 3);
-
- // Step 1: general dimension info inputs
- long worstCaseDim = -1;
- // since the dimensions of both inputs must be the same, checking for one input is sufficient
- if( mc[0].dimsKnown() || mc[1].dimsKnown() ) {
- // Output dimensions are completely data dependent. In the worst case,
- // #categories in each attribute = #rows (e.g., an ID column, say EmployeeID).
- // both inputs are one-dimensional matrices with exact same dimensions, m = size of longer dimension
- worstCaseDim = (mc[0].dimsKnown())
- ? (mc[0].getRows() > 1 ? mc[0].getRows() : mc[0].getCols() )
- : (mc[1].getRows() > 1 ? mc[1].getRows() : mc[1].getCols() );
- //note: for ctable histogram dim2 known but automatically replaces m
- //ret = new long[]{m, m, m};
- }
-
- // Step 2: special handling specified dims
- if( dimsSpec && getInput().get(3) instanceof LiteralOp && getInput().get(4) instanceof LiteralOp )
- {
- long outputDim1 = HopRewriteUtils.getIntValueSafe((LiteralOp)getInput().get(3));
- long outputDim2 = HopRewriteUtils.getIntValueSafe((LiteralOp)getInput().get(4));
- long outputNNZ = ( outputDim1*outputDim2 > outputDim1 ? outputDim1 : outputDim1*outputDim2 );
- _dim1 = outputDim1;
- _dim2 = outputDim2;
- return new long[]{outputDim1, outputDim2, outputNNZ};
- }
-
- // Step 3: general case
- //note: for ctable histogram dim2 known but automatically replaces m
- return new long[]{worstCaseDim, worstCaseDim, worstCaseDim};
-
- case QUANTILE:
- if( mc[2].dimsKnown() )
- return new long[]{mc[2].getRows(), 1, mc[2].getRows()};
- break;
-
- default:
- throw new RuntimeException("Memory for operation (" + _op + ") can not be estimated.");
- }
-
- return ret;
- }
-
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if ( (getInput().get(0).areDimsBelowThreshold()
- && getInput().get(1).areDimsBelowThreshold()
- && getInput().get(2).areDimsBelowThreshold())
- //|| (getInput().get(0).isVector() && getInput().get(1).isVector() && getInput().get(1).isVector() )
- )
- _etype = ExecType.CP;
- else
- _etype = REMOTE;
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- // Necessary condition for recompilation is unknown dimensions.
- // When execType=CP, it is marked for recompilation only when additional
- // dimension inputs are provided (and those values are unknown at initial compile time).
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) ) {
- if ( _etype==REMOTE || (_etype == ExecType.CP && _dimInputsPresent))
- setRequiresRecompile();
- }
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- if ( getDataType() == DataType.SCALAR )
- {
- //do nothing always known
- }
- else
- {
- switch( _op )
- {
- case CTABLE:
- //in general, do nothing because the output size is data dependent
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- Hop input3 = getInput().get(2);
-
-
- if ( _dim1 == -1 || _dim2 == -1 ) {
- //for ctable_expand at least one dimension is known
- if( isSequenceRewriteApplicable() )
- {
- if( input1 instanceof DataGenOp && ((DataGenOp)input1).getOp()==DataGenMethod.SEQ )
- setDim1( input1._dim1 );
- else //if( input2 instanceof DataGenOp && ((DataGenOp)input2).getDataGenMethod()==DataGenMethod.SEQ )
- setDim2( input2._dim1 );
- }
- //for ctable_histogram also one dimension is known
- Ternary.OperationTypes tertiaryOp = Ternary.findCtableOperationByInputDataTypes(
- input1.getDataType(), input2.getDataType(), input3.getDataType());
- if( tertiaryOp==Ternary.OperationTypes.CTABLE_TRANSFORM_HISTOGRAM
- && input2 instanceof LiteralOp )
- {
- setDim2( HopRewriteUtils.getIntValueSafe((LiteralOp)input2) );
- }
-
- // if output dimensions are provided, update _dim1 and _dim2
- if( getInput().size() >= 5 ) {
- if( getInput().get(3) instanceof LiteralOp )
- setDim1( HopRewriteUtils.getIntValueSafe((LiteralOp)getInput().get(3)) );
- if( getInput().get(4) instanceof LiteralOp )
- setDim2( HopRewriteUtils.getIntValueSafe((LiteralOp)getInput().get(4)) );
- }
- }
-
- break;
-
- case QUANTILE:
- // This part of the code is executed only when a vector of quantiles are computed
- // Output is a vector of length = #of quantiles to be computed, and it is likely to be dense.
- // TODO qx1
- break;
-
- default:
- throw new RuntimeException("Size information for operation (" + _op + ") can not be updated.");
- }
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- TernaryOp ret = new TernaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._dimInputsPresent = _dimInputsPresent;
- ret._disjointInputs = _disjointInputs;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof TernaryOp) )
- return false;
-
- TernaryOp that2 = (TernaryOp)that;
-
- //compare basic inputs and weights (always existing)
- boolean ret = (_op == that2._op
- && getInput().get(0) == that2.getInput().get(0)
- && getInput().get(1) == that2.getInput().get(1)
- && getInput().get(2) == that2.getInput().get(2));
-
- //compare optional dimension parameters
- ret &= (_dimInputsPresent == that2._dimInputsPresent);
- if( ret && _dimInputsPresent ){
- ret &= getInput().get(3) == that2.getInput().get(3)
- && getInput().get(4) == that2.getInput().get(4);
- }
-
- //compare optimizer hints and parameters
- ret &= _disjointInputs == that2._disjointInputs
- && _outputEmptyBlocks == that2._outputEmptyBlocks;
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- private boolean isSequenceRewriteApplicable()
- {
- return isSequenceRewriteApplicable(true)
- || isSequenceRewriteApplicable(false);
- }
-
- /**
- *
- * @param left
- * @return
- */
- private boolean isSequenceRewriteApplicable( boolean left )
- {
- boolean ret = false;
-
- //early abort if rewrite globally not allowed
- if( !ALLOW_CTABLE_SEQUENCE_REWRITES )
- return ret;
-
- try
- {
- if( getInput().size()==2 || (getInput().size()==3 && getInput().get(2).getDataType()==DataType.SCALAR) )
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- if( input1.getDataType() == DataType.MATRIX && input2.getDataType() == DataType.MATRIX )
- {
- //probe rewrite on left input
- if( left && input1 instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) input1;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop incr = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_INCR));
- ret = (incr instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)incr)==1)
- || dgop.getIncrementValue()==1.0; //set by recompiler
- }
- }
- //probe rewrite on right input
- if( !left && input2 instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) input2;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop incr = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_INCR));
- ret |= (incr instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)incr)==1)
- || dgop.getIncrementValue()==1.0; //set by recompiler;
- }
- }
- }
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- //ret = false;
- }
-
- return ret;
- }
-
- /**
- * Used for (1) constructing CP lops (hop-lop rewrite), and (2) in order to determine
- * if dag split after removeEmpty necessary (#2 is precondition for #1).
- *
- * @return
- */
- public boolean isMatrixIgnoreZeroRewriteApplicable()
- {
- boolean ret = false;
-
- //early abort if rewrite globally not allowed
- if( !ALLOW_CTABLE_SEQUENCE_REWRITES || _op!=OpOp3.CTABLE )
- return ret;
-
- try
- {
- //1) check for ctable CTABLE_TRANSFORM_SCALAR_WEIGHT
- if( getInput().size()==2 || (getInput().size()>2 && getInput().get(2).getDataType()==DataType.SCALAR) )
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- //2) check for remove empty pair
- if( input1.getDataType() == DataType.MATRIX && input2.getDataType() == DataType.MATRIX
- && input1 instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp)input1).getOp()==ParamBuiltinOp.RMEMPTY
- && input2 instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp)input2).getOp()==ParamBuiltinOp.RMEMPTY )
- {
- ParameterizedBuiltinOp pb1 = (ParameterizedBuiltinOp)input1;
- ParameterizedBuiltinOp pb2 = (ParameterizedBuiltinOp)input2;
- Hop pbin1 = pb1.getTargetHop();
- Hop pbin2 = pb2.getTargetHop();
-
- //3) check for reshape pair
- if( pbin1 instanceof ReorgOp && ((ReorgOp)pbin1).getOp()==ReOrgOp.RESHAPE
- && pbin2 instanceof ReorgOp && ((ReorgOp)pbin2).getOp()==ReOrgOp.RESHAPE )
- {
- //4) check common non-zero input (this allows to infer two things:
- //(a) that the dims are equivalent, and zero values for remove empty are aligned)
- Hop left = pbin1.getInput().get(0);
- Hop right = pbin2.getInput().get(0);
- if( left instanceof BinaryOp && ((BinaryOp)left).getOp()==OpOp2.MULT
- && left.getInput().get(0) instanceof BinaryOp && ((BinaryOp)left.getInput().get(0)).getOp()==OpOp2.NOTEQUAL
- && left.getInput().get(0).getInput().get(1) instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)left.getInput().get(0).getInput().get(1))==0
- && left.getInput().get(0).getInput().get(0) == right ) //relies on CSE
- {
- ret = true;
- }
- else if( right instanceof BinaryOp && ((BinaryOp)right).getOp()==OpOp2.MULT
- && right.getInput().get(0) instanceof BinaryOp && ((BinaryOp)right.getInput().get(0)).getOp()==OpOp2.NOTEQUAL
- && right.getInput().get(0).getInput().get(1) instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)right.getInput().get(0).getInput().get(1))==0
- && right.getInput().get(0).getInput().get(0) == left ) //relies on CSE
- {
- ret = true;
- }
- }
- }
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- //ret = false;
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/UnaryOp.java b/src/main/java/com/ibm/bi/dml/hops/UnaryOp.java
deleted file mode 100644
index 71af54f..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/UnaryOp.java
+++ /dev/null
@@ -1,746 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.CombineUnary;
-import com.ibm.bi.dml.lops.CumulativeOffsetBinary;
-import com.ibm.bi.dml.lops.CumulativePartialAggregate;
-import com.ibm.bi.dml.lops.CumulativeSplitAggregate;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-
-/* Unary (cell operations): e.g, b_ij = round(a_ij)
- * Semantic: given a value, perform the operation (independent of other values)
- */
-
-public class UnaryOp extends Hop
-{
-
- private OpOp1 _op = null;
-
-
- private UnaryOp() {
- //default constructor for clone
- }
-
- public UnaryOp(String l, DataType dt, ValueType vt, OpOp1 o, Hop inp)
- throws HopsException
- {
- super(l, dt, vt);
-
- getInput().add(0, inp);
- inp.getParent().add(this);
-
- _op = o;
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- // this is for OpOp1, e.g. A = -B (0-B); and a=!b
- public OpOp1 getOp() {
- return _op;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + _op);
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += "u(" + _op.toString().toLowerCase() + ")";
- // s += HopsOpOp1String.get(_op) + ")";
- return s;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //reuse existing lop
- if( getLops() != null )
- return getLops();
-
- try
- {
- Hop input = getInput().get(0);
-
- if (getDataType() == DataType.SCALAR || _op == OpOp1.CAST_AS_MATRIX)
- {
- if (_op == Hop.OpOp1.IQM) //special handling IQM
- {
- Lop iqmLop = constructLopsIQM();
- setLops(iqmLop);
- }
- else if(_op == Hop.OpOp1.MEDIAN) {
- Lop medianLop = constructLopsMedian();
- setLops(medianLop);
- }
- else //general case SCALAR/CAST (always in CP)
- {
- UnaryCP.OperationTypes optype = HopsOpOp1LopsUS.get(_op);
- if( optype == null )
- throw new HopsException("Unknown UnaryCP lop type for UnaryOp operation type '"+_op+"'");
-
- UnaryCP unary1 = new UnaryCP(input.constructLops(), optype, getDataType(), getValueType());
- setOutputDimensions(unary1);
- setLineNumbers(unary1);
-
- setLops(unary1);
- }
- }
- else //general case MATRIX
- {
- ExecType et = optFindExecType();
-
- //special handling cumsum/cumprod/cummin/cumsum
- if( isCumulativeUnaryOperation() && et != ExecType.CP )
- {
- //TODO additional physical operation if offsets fit in memory
- Lop cumsumLop = null;
- if( et == ExecType.MR )
- cumsumLop = constructLopsMRCumulativeUnary();
- else
- cumsumLop = constructLopsSparkCumulativeUnary();
- setLops(cumsumLop);
- }
- else //default unary
- {
- Unary unary1 = new Unary(input.constructLops(), HopsOpOp1LopsU.get(_op),
- getDataType(), getValueType(), et);
- setOutputDimensions(unary1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- }
- }
- catch (Exception e)
- {
- throw new HopsException(this.printErrorLocation() + "error constructing Lops for UnaryOp Hop -- \n " , e);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
-
- private Lop constructLopsMedian()
- throws HopsException, LopsException
- {
- ExecType et = optFindExecType();
-
- if ( et == ExecType.MR ) {
- CombineUnary combine = CombineUnary.constructCombineLop(
- getInput().get(0).constructLops(),
- getDataType(), getValueType());
-
- SortKeys sort = SortKeys.constructSortByValueLop(
- combine, SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, et);
-
- combine.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- // Sort dimensions are same as the first input
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- // If only a single quantile is computed, then "pick" operation executes in CP.
- ExecType et_pick = ExecType.CP;
-
- PickByCount pick = new PickByCount(
- sort,
- Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)),
- getDataType(),
- getValueType(),
- PickByCount.OperationTypes.MEDIAN, et_pick, false);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pick);
-
- return pick;
- }
- else {
- SortKeys sort = SortKeys.constructSortByValueLop(
- getInput().get(0).constructLops(),
- SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, et );
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
- PickByCount pick = new PickByCount(
- sort,
- Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)),
- getDataType(),
- getValueType(),
- PickByCount.OperationTypes.MEDIAN, et, true);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pick);
- setLops(pick);
-
- return pick;
- }
- }
-
- private Lop constructLopsIQM()
- throws HopsException, LopsException
- {
-
- ExecType et = optFindExecType();
-
- Hop input = getInput().get(0);
- if ( et == ExecType.MR ) {
- CombineUnary combine = CombineUnary.constructCombineLop(input.constructLops(),
- DataType.MATRIX, getValueType());
- combine.getOutputParameters().setDimensions(
- input.getDim1(),
- input.getDim2(),
- input.getRowsInBlock(),
- input.getColsInBlock(),
- input.getNnz());
-
- SortKeys sort = SortKeys.constructSortByValueLop(combine,
- SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
-
- // Sort dimensions are same as the first input
- sort.getOutputParameters().setDimensions(
- input.getDim1(),
- input.getDim2(),
- input.getRowsInBlock(),
- input.getColsInBlock(),
- input.getNnz());
-
- Data lit = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.25));
-
- lit.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- PickByCount pick = new PickByCount(
- sort, lit, DataType.MATRIX, getValueType(),
- PickByCount.OperationTypes.RANGEPICK);
-
- pick.getOutputParameters().setDimensions(-1, -1,
- getRowsInBlock(), getColsInBlock(), -1);
- setLineNumbers(pick);
-
- PartialAggregate pagg = new PartialAggregate(
- pick, HopsAgg2Lops.get(Hop.AggOp.SUM),
- HopsDirection2Lops.get(Hop.Direction.RowCol),
- DataType.MATRIX, getValueType());
- setLineNumbers(pagg);
-
- // Set the dimensions of PartialAggregate LOP based on the
- // direction in which aggregation is performed
- pagg.setDimensionsBasedOnDirection(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock());
-
- Group group1 = new Group(
- pagg, Group.OperationTypes.Sort, DataType.MATRIX,
- getValueType());
- group1.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- setLineNumbers(group1);
-
- Aggregate agg1 = new Aggregate(
- group1, HopsAgg2Lops.get(Hop.AggOp.SUM),
- DataType.MATRIX, getValueType(), ExecType.MR);
- agg1.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(),
- getColsInBlock(), getNnz());
- agg1.setupCorrectionLocation(pagg.getCorrectionLocation());
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(
- agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
-
- Unary iqm = new Unary(sort, unary1, Unary.OperationTypes.MR_IQM, DataType.SCALAR, ValueType.DOUBLE, ExecType.CP);
- iqm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(iqm);
-
- return iqm;
- }
- else {
- SortKeys sort = SortKeys.constructSortByValueLop(
- input.constructLops(),
- SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, et );
- sort.getOutputParameters().setDimensions(
- input.getDim1(),
- input.getDim2(),
- input.getRowsInBlock(),
- input.getColsInBlock(),
- input.getNnz());
- PickByCount pick = new PickByCount(sort, null,
- getDataType(),getValueType(),
- PickByCount.OperationTypes.IQM, et, true);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
- setLineNumbers(pick);
-
- return pick;
- }
- }
-
- /**
- * MR Cumsum is currently based on a multipass algorithm of (1) preaggregation and (2) subsequent offsetting.
- * Note that we currently support one robust physical operator but many alternative
- * realizations are possible for specific scenarios (e.g., when the preaggregated intermediate
- * fit into the map task memory budget) or by creating custom job types.
- *
- *
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructLopsMRCumulativeUnary()
- throws HopsException, LopsException
- {
- Hop input = getInput().get(0);
- long rlen = input.getDim1();
- long clen = input.getDim2();
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- boolean force = !dimsKnown() || _etypeForced == ExecType.MR;
- OperationTypes aggtype = getCumulativeAggType();
-
- Lop X = input.constructLops();
- Lop TEMP = X;
- ArrayList<Lop> DATA = new ArrayList<Lop>();
- int level = 0;
-
- //recursive preaggregation until aggregates fit into CP memory budget
- while( ((2*OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen) + OptimizerUtils.estimateSize(1, clen))
- > OptimizerUtils.getLocalMemBudget()
- && TEMP.getOutputParameters().getNumRows()>1) || force )
- {
- DATA.add(TEMP);
-
- //preaggregation per block
- long rlenAgg = (long)Math.ceil((double)TEMP.getOutputParameters().getNumRows()/brlen);
- Lop preagg = new CumulativePartialAggregate(TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR);
- preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
- setLineNumbers(preagg);
-
- Group group = new Group( preagg, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE );
- group.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
- setLineNumbers(group);
-
- Aggregate agg = new Aggregate(group, HopsAgg2Lops.get(AggOp.SUM), getDataType(), getValueType(), ExecType.MR);
- agg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
- agg.setupCorrectionLocation(CorrectionLocationType.NONE); // aggregation uses kahanSum but the inputs do not have correction values
- setLineNumbers(agg);
- TEMP = agg;
- level++;
- force = false; //in case of unknowns, generate one level
- }
-
- //in-memory cum sum (of partial aggregates)
- if( TEMP.getOutputParameters().getNumRows()!=1 ){
- Unary unary1 = new Unary( TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP);
- unary1.getOutputParameters().setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1);
- setLineNumbers(unary1);
- TEMP = unary1;
- }
-
- //split, group and mr cumsum
- while( level-- > 0 ) {
- double init = getCumulativeInitValue();
- CumulativeSplitAggregate split = new CumulativeSplitAggregate(TEMP, DataType.MATRIX, ValueType.DOUBLE, init);
- split.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
- setLineNumbers(split);
-
- Group group1 = new Group( DATA.get(level), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE );
- group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
- setLineNumbers(group1);
-
- Group group2 = new Group( split, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE );
- group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
- setLineNumbers(group2);
-
- CumulativeOffsetBinary binary = new CumulativeOffsetBinary(group1, group2,
- DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR);
- binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
- setLineNumbers(binary);
- TEMP = binary;
- }
-
- return TEMP;
- }
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructLopsSparkCumulativeUnary()
- throws HopsException, LopsException
- {
- Hop input = getInput().get(0);
- long rlen = input.getDim1();
- long clen = input.getDim2();
- long brlen = input.getRowsInBlock();
- long bclen = input.getColsInBlock();
- boolean force = !dimsKnown() || _etypeForced == ExecType.SPARK;
- OperationTypes aggtype = getCumulativeAggType();
-
- Lop X = input.constructLops();
- Lop TEMP = X;
- ArrayList<Lop> DATA = new ArrayList<Lop>();
- int level = 0;
-
- //recursive preaggregation until aggregates fit into CP memory budget
- while( ((2*OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen) + OptimizerUtils.estimateSize(1, clen))
- > OptimizerUtils.getLocalMemBudget()
- && TEMP.getOutputParameters().getNumRows()>1) || force )
- {
- DATA.add(TEMP);
-
- //preaggregation per block (for spark, the CumulativePartialAggregate subsumes both
- //the preaggregation and subsequent block aggregation)
- long rlenAgg = (long)Math.ceil((double)TEMP.getOutputParameters().getNumRows()/brlen);
- Lop preagg = new CumulativePartialAggregate(TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.SPARK);
- preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
- setLineNumbers(preagg);
-
- TEMP = preagg;
- level++;
- force = false; //in case of unknowns, generate one level
- }
-
- //in-memory cum sum (of partial aggregates)
- if( TEMP.getOutputParameters().getNumRows()!=1 ){
- Unary unary1 = new Unary( TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP);
- unary1.getOutputParameters().setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1);
- setLineNumbers(unary1);
- TEMP = unary1;
- }
-
- //split, group and mr cumsum
- while( level-- > 0 ) {
- //(for spark, the CumulativeOffsetBinary subsumes both the split aggregate and
- //the subsequent offset binary apply of split aggregates against the original data)
- double initValue = getCumulativeInitValue();
- CumulativeOffsetBinary binary = new CumulativeOffsetBinary(DATA.get(level), TEMP,
- DataType.MATRIX, ValueType.DOUBLE, initValue, aggtype, ExecType.SPARK);
- binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
- setLineNumbers(binary);
- TEMP = binary;
- }
-
- return TEMP;
- }
-
- /**
- *
- * @return
- */
- private OperationTypes getCumulativeAggType()
- {
- switch( _op ) {
- case CUMSUM: return OperationTypes.KahanSum;
- case CUMPROD: return OperationTypes.Product;
- case CUMMIN: return OperationTypes.Min;
- case CUMMAX: return OperationTypes.Max;
- default: return null;
- }
- }
-
- /**
- *
- * @return
- */
- private double getCumulativeInitValue()
- {
- switch( _op ) {
- case CUMSUM: return 0;
- case CUMPROD: return 1;
- case CUMMIN: return Double.MAX_VALUE;
- case CUMMAX: return -Double.MAX_VALUE;
- default: return Double.NaN;
- }
- }
-
-
- @Override
- public void computeMemEstimate(MemoTable memo)
- {
- //overwrites default hops behavior
- super.computeMemEstimate(memo);
-
- if( _op == Hop.OpOp1.NROW || _op == Hop.OpOp1.NCOL ) //specific case for meta data ops
- {
- _memEstimate = OptimizerUtils.INT_SIZE;
- //_outputMemEstimate = OptimizerUtils.INT_SIZE;
- //_processingMemEstimate = 0;
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if ( _op == OpOp1.IQM || _op == OpOp1.MEDIAN) {
- // buffer (=2*input_size) and output (=input_size) for SORT operation
- // getMemEstimate works for both cases of known dims and worst-case stats
- ret = getInput().get(0).getMemEstimate() * 3;
- }
-
- return ret;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- Hop input = getInput().get(0);
- MatrixCharacteristics mc = memo.getAllInputStats(input);
- if( mc.dimsKnown() ) {
- if( _op==OpOp1.ABS || _op==OpOp1.COS || _op==OpOp1.SIN || _op==OpOp1.TAN
- || _op==OpOp1.ACOS || _op==OpOp1.ASIN || _op==OpOp1.ATAN
- || _op==OpOp1.SQRT || _op==OpOp1.ROUND
- || _op==OpOp1.SPROP || _op==OpOp1.SELP ) //sparsity preserving
- {
- ret = new long[]{mc.getRows(), mc.getCols(), mc.getNonZeros()};
- }
- else
- ret = new long[]{mc.getRows(), mc.getCols(), -1};
- }
-
- return ret;
- }
-
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- /**
- *
- * @return
- */
- private boolean isInMemoryOperation()
- {
- return ( _op == OpOp1.INVERSE );
- }
-
- /**
- *
- * @return
- */
- public boolean isCumulativeUnaryOperation()
- {
- return ( _op == OpOp1.CUMSUM
- || _op == OpOp1.CUMPROD
- || _op == OpOp1.CUMMIN
- || _op == OpOp1.CUMMAX );
- }
-
- /**
- *
- * @return
- */
- public boolean isCastUnaryOperation()
- {
- return ( _op == OpOp1.CAST_AS_MATRIX
- || _op == OpOp1.CAST_AS_SCALAR
- || _op == OpOp1.CAST_AS_BOOLEAN
- || _op == OpOp1.CAST_AS_DOUBLE
- || _op == OpOp1.CAST_AS_INT );
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- // Choose CP, if the input dimensions are below threshold or if the input is a vector
- // Also, matrix inverse is currently implemented only in CP (through commons math)
- else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()
- || isInMemoryOperation() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //spark-specific decision refinement (execute unary w/ spark input and
- //single parent also in spark because it's likely cheap and reduces intermediates)
- if( _etype == ExecType.CP && _etypeForced != ExecType.CP
- && getInput().get(0).optFindExecType() == ExecType.SPARK
- && getDataType().isMatrix()
- && !isCumulativeUnaryOperation() && !isCastUnaryOperation()
- && _op!=OpOp1.MEDIAN && _op!=OpOp1.IQM
- && !(getInput().get(0) instanceof DataOp) //input is not checkpoint
- && getInput().get(0).getParent().size()==1 ) //unary is only parent
- {
- //pull unary operation into spark
- _etype = ExecType.SPARK;
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- //ensure cp exec type for single-node operations
- if( _op == OpOp1.PRINT || _op == OpOp1.STOP || _op == OpOp1.INVERSE || _op == OpOp1.EIGEN )
- _etype = ExecType.CP;
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- if ( getDataType() == DataType.SCALAR )
- {
- //do nothing always known
- }
- else if( _op == OpOp1.CAST_AS_MATRIX && getInput().get(0).getDataType()==DataType.SCALAR )
- {
- //prevent propagating 0 from scalar (which would be interpreted as unknown)
- setDim1( 1 );
- setDim2( 1 );
- }
- else //general case
- {
- // If output is a Matrix then this operation is of type (B = op(A))
- // Dimensions of B are same as that of A, and sparsity may/maynot change
- Hop input = getInput().get(0);
- setDim1( input.getDim1() );
- setDim2( input.getDim2() );
- if( _op==OpOp1.ABS || _op==OpOp1.COS || _op==OpOp1.SIN || _op==OpOp1.TAN
- || _op==OpOp1.ACOS || _op==OpOp1.ASIN || _op==OpOp1.ATAN
- || _op==OpOp1.SQRT || _op==OpOp1.ROUND || _op==OpOp1.SPROP ) //sparsity preserving
- {
- setNnz( input.getNnz() );
- }
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- UnaryOp ret = new UnaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof UnaryOp) )
- return false;
-
- /*
- * NOTE:
- * This compare() method currently is invoked from Hops RewriteCommonSubexpressionElimination,
- * which tries to merge two hops if this function returns true. However, two PRINT hops should
- * never be merged, and hence returning false.
- *
- * If this method needs to be used elsewhere, then it must be refactored accordingly.
- */
- if( _op == OpOp1.PRINT )
- return false;
-
- UnaryOp that2 = (UnaryOp)that;
- return ( _op == that2._op
- && getInput().get(0) == that2.getInput().get(0));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimationWrapper.java b/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimationWrapper.java
deleted file mode 100644
index b4c669b..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/cost/CostEstimationWrapper.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.cost;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-
-public class CostEstimationWrapper
-{
-
- public enum CostType {
- NUM_MRJOBS, //based on number of MR jobs, [number MR jobs]
- STATIC, // based on FLOPS, read/write, etc, [time in sec]
- DYNAMIC // based on dynamic offline performance profile, [time in sec]
- };
-
- private static final boolean LDEBUG = false; //internal local debug level
- private static final Log LOG = LogFactory.getLog(CostEstimationWrapper.class.getName());
- private static final CostType DEFAULT_COSTTYPE = CostType.STATIC;
-
- private static CostEstimator _costEstim = null;
-
-
- static
- {
- // for internal debugging only
- if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.cost")
- .setLevel((Level) Level.DEBUG);
- }
-
- //create cost estimator
- try
- {
- //TODO config parameter?
- _costEstim = createCostEstimator(DEFAULT_COSTTYPE);
- }
- catch(Exception ex)
- {
- LOG.error("Failed cost estimator initialization.", ex);
- }
- }
-
- /**
- *
- * @param rtprog
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static double getTimeEstimate(Program rtprog, ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- Timing time = new Timing(true);
-
- HashMap<String,VarStats> stats = new HashMap<String, VarStats>();
- LocalVariableMap vars = (ec!=null)? ec.getVariables() : new LocalVariableMap();
-
- double costs = _costEstim.getTimeEstimate(rtprog, vars, stats);
- LOG.debug("Finished estimation in "+time.stop()+"ms.");
- return costs;
- }
-
- /**
- *
- * @param pb
- * @param ec
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public static double getTimeEstimate(ProgramBlock pb, ExecutionContext ec, boolean recursive)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- Timing time = new Timing(true);
-
- HashMap<String,VarStats> stats = new HashMap<String, VarStats>();
- LocalVariableMap vars = (ec!=null)? ec.getVariables() : new LocalVariableMap();
-
- double costs = _costEstim.getTimeEstimate(pb, vars, stats, recursive);
- LOG.debug("Finished estimation in "+time.stop()+"ms.");
- return costs;
- }
-
- /**
- *
- * @param hops
- * @param ec
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- * @throws HopsException
- * @throws LopsException
- * @throws IOException
- */
- public static double getTimeEstimate( ArrayList<Hop> hops, ExecutionContext ec )
- throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException, LopsException, IOException
- {
- Timing time = new Timing(true);
-
- HashMap<String,VarStats> stats = new HashMap<String, VarStats>();
- LocalVariableMap vars = (ec!=null)? ec.getVariables() : new LocalVariableMap();
-
- double costs = _costEstim.getTimeEstimate(hops, vars, stats);
- LOG.debug("Finished estimation in "+time.stop()+"ms.");
-
- return costs;
- }
-
- /**
- *
- * @param type
- * @return
- * @throws DMLRuntimeException
- */
- private static CostEstimator createCostEstimator( CostType type )
- throws DMLRuntimeException
- {
- switch( type )
- {
- case NUM_MRJOBS:
- return new CostEstimatorNumMRJobs();
- case STATIC:
- return new CostEstimatorStaticRuntime();
- default:
- throw new DMLRuntimeException("Unknown cost type: "+type);
- }
- }
-}
[20/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DataExpression.java b/src/main/java/com/ibm/bi/dml/parser/DataExpression.java
deleted file mode 100644
index d64910e..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DataExpression.java
+++ /dev/null
@@ -1,2171 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.wink.json4j.JSONArray;
-import org.apache.wink.json4j.JSONObject;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.JSONHelper;
-
-
-public class DataExpression extends DataIdentifier
-{
-
- public static boolean REJECT_READ_UNKNOWN_SIZE = true;
-
- public static final String RAND_ROWS = "rows";
- public static final String RAND_COLS = "cols";
- public static final String RAND_MIN = "min";
- public static final String RAND_MAX = "max";
- public static final String RAND_SPARSITY = "sparsity";
- public static final String RAND_SEED = "seed";
- public static final String RAND_PDF = "pdf";
- public static final String RAND_LAMBDA = "lambda";
-
- public static final String RAND_PDF_UNIFORM = "uniform";
-
- public static final String RAND_BY_ROW = "byrow";
- public static final String RAND_DIMNAMES = "dimnames";
- public static final String RAND_DATA = "data";
-
- public static final String IO_FILENAME = "iofilename";
- public static final String READROWPARAM = "rows";
- public static final String READCOLPARAM = "cols";
- public static final String READNUMNONZEROPARAM = "nnz";
-
- public static final String FORMAT_TYPE = "format";
- public static final String FORMAT_TYPE_VALUE_TEXT = "text";
- public static final String FORMAT_TYPE_VALUE_BINARY = "binary";
- public static final String FORMAT_TYPE_VALUE_CSV = "csv";
- public static final String FORMAT_TYPE_VALUE_MATRIXMARKET = "mm";
-
- public static final String ROWBLOCKCOUNTPARAM = "rows_in_block";
- public static final String COLUMNBLOCKCOUNTPARAM = "cols_in_block";
- public static final String DATATYPEPARAM = "data_type";
- public static final String VALUETYPEPARAM = "value_type";
- public static final String DESCRIPTIONPARAM = "description";
-
- // Parameter names relevant to reading/writing delimited/csv files
- public static final String DELIM_DELIMITER = "sep";
- public static final String DELIM_HAS_HEADER_ROW = "header";
- public static final String DELIM_FILL = "fill";
- public static final String DELIM_FILL_VALUE = "default";
- //public static final String DELIM_RECODE = "recode";
- public static final String DELIM_NA_STRINGS = "na.strings";
- public static final String DELIM_NA_STRING_SEP = "\u00b7";
-
- public static final String DELIM_SPARSE = "sparse"; // applicable only for write
-
- public static final String[] RAND_VALID_PARAM_NAMES =
- { RAND_ROWS, RAND_COLS, RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, RAND_PDF, RAND_LAMBDA};
-
- public static final String[] MATRIX_VALID_PARAM_NAMES =
- { RAND_BY_ROW, RAND_DIMNAMES, RAND_DATA, RAND_ROWS, RAND_COLS};
-
- // Valid parameter names in a metadata file
- public static final String[] READ_VALID_MTD_PARAM_NAMES =
- { IO_FILENAME, READROWPARAM, READCOLPARAM, READNUMNONZEROPARAM, FORMAT_TYPE,
- ROWBLOCKCOUNTPARAM, COLUMNBLOCKCOUNTPARAM, DATATYPEPARAM, VALUETYPEPARAM, DESCRIPTIONPARAM,
- // Parameters related to delimited/csv files.
- DELIM_FILL_VALUE, DELIM_DELIMITER, DELIM_FILL, DELIM_HAS_HEADER_ROW, DELIM_NA_STRINGS
- };
-
- public static final String[] READ_VALID_PARAM_NAMES =
- { IO_FILENAME, READROWPARAM, READCOLPARAM, FORMAT_TYPE, DATATYPEPARAM, VALUETYPEPARAM,
- ROWBLOCKCOUNTPARAM, COLUMNBLOCKCOUNTPARAM, READNUMNONZEROPARAM,
- // Parameters related to delimited/csv files.
- DELIM_FILL_VALUE, DELIM_DELIMITER, DELIM_FILL, DELIM_HAS_HEADER_ROW, DELIM_NA_STRINGS
- };
-
- /* Default Values for delimited (CSV) files */
- public static final String DEFAULT_DELIM_DELIMITER = ",";
- public static final boolean DEFAULT_DELIM_HAS_HEADER_ROW = false;
- public static final boolean DEFAULT_DELIM_FILL = true;
- public static final double DEFAULT_DELIM_FILL_VALUE = 0.0;
- public static final boolean DEFAULT_DELIM_SPARSE = false;
-
- private DataOp _opcode;
- private HashMap<String, Expression> _varParams;
- private boolean _strInit = false; //string initialize
-
- private boolean checkMetadata = true;
- public void setCheckMetadata(boolean checkMetadata) {
- this.checkMetadata = checkMetadata;
- }
-
- public DataExpression(){
- //do nothing
- }
-
- public static DataExpression getDataExpression(String functionName, ArrayList<ParameterExpression> passedParamExprs,
- String filename, int blp, int bcp, int elp, int ecp) throws DMLParseException {
-
- if (functionName == null || passedParamExprs == null)
- return null;
-
- // check if the function name is built-in function
- // (assign built-in function op if function is built-in)
- Expression.DataOp dop = null;
- DataExpression dataExpr = null;
- if (functionName.equals("read") || functionName.equals("readMM") || functionName.equals("read.csv"))
- {
- dop = Expression.DataOp.READ;
- dataExpr = new DataExpression(dop, new HashMap<String,Expression>(), filename, blp, bcp, elp, ecp);
-
- if (functionName.equals("readMM"))
- dataExpr.addVarParam(DataExpression.FORMAT_TYPE,
- new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET,
- filename, blp, bcp, elp, ecp));
-
- if (functionName.equals("read.csv"))
- dataExpr.addVarParam(DataExpression.FORMAT_TYPE,
- new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV,
- filename, blp, bcp, elp, ecp));
-
- // validate the filename is the first parameter
- if (passedParamExprs.size() < 1){
- // throw exception -- must be filename as first parameter
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "read method must have at least filename parameter");
- }
-
- ParameterExpression pexpr = (passedParamExprs.size() == 0) ? null : passedParamExprs.get(0);
-
- if ( (pexpr != null) && (!(pexpr.getName() == null) || (pexpr.getName() != null && pexpr.getName().equalsIgnoreCase(DataExpression.IO_FILENAME)))){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "first parameter to read statement must be filename");
- } else if( pexpr != null ){
- dataExpr.addVarParam(DataExpression.IO_FILENAME, pexpr.getExpr());
- }
-
- // validate all parameters are added only once and valid name
- for (int i = 1; i < passedParamExprs.size(); i++){
- String currName = passedParamExprs.get(i).getName();
- Expression currExpr = passedParamExprs.get(i).getExpr();
-
- if (dataExpr.getVarParam(currName) != null){
- throw new DMLParseException(dataExpr.getFilename(), currExpr.printErrorLocation() + "attempted to add IOStatement parameter " + currName + " more than once");
- }
- // verify parameter names for read function
- boolean isValidName = false;
- for (String paramName : READ_VALID_PARAM_NAMES){
- if (paramName.equals(currName))
- isValidName = true;
- }
- if (!isValidName){
- throw new DMLParseException(dataExpr.getFilename(), currExpr.printErrorLocation() + "attempted to add invalid read statement parameter " + currName);
- }
- dataExpr.addVarParam(currName, currExpr);
- }
- }
-
- else if (functionName.equalsIgnoreCase("rand")){
-
- dop = Expression.DataOp.RAND;
- dataExpr = new DataExpression(dop, new HashMap<String,Expression>(),
- filename, blp, bcp, elp, ecp);
-
- for (ParameterExpression currExpr : passedParamExprs){
- String pname = currExpr.getName();
- Expression pexpr = currExpr.getExpr();
- if (pname == null){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for Rand Statment all arguments must be named parameters");
- //LOG.error(dataExpr.printErrorLocation(beginLine, beginColumn) + "for Rand Statment all arguments must be named parameters");
- //throw new ParseException(dataExpr.printErrorLocation(beginLine, beginColumn) + "for Rand Statment all arguments must be named parameters");
- }
- dataExpr.addRandExprParam(pname, pexpr);
- }
- dataExpr.setRandDefault();
- }
-
- else if (functionName.equals("matrix")){
- dop = Expression.DataOp.MATRIX;
- dataExpr = new DataExpression(dop, new HashMap<String,Expression>(),
- filename, blp, bcp, elp, ecp);
-
- int namedParamCount = 0, unnamedParamCount = 0;
- for (ParameterExpression currExpr : passedParamExprs) {
- if (currExpr.getName() == null)
- unnamedParamCount++;
- else
- namedParamCount++;
- }
-
- // check whether named or unnamed parameters are used
- if (passedParamExprs.size() < 3){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
- }
-
- if (unnamedParamCount > 1){
-
- if (namedParamCount > 0)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, cannot mix named and unnamed parameters");
-
- if (unnamedParamCount < 3)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
-
-
- // assume: data, rows, cols, [byRow], [dimNames]
- dataExpr.addMatrixExprParam(DataExpression.RAND_DATA,passedParamExprs.get(0).getExpr());
- dataExpr.addMatrixExprParam(DataExpression.RAND_ROWS,passedParamExprs.get(1).getExpr());
- dataExpr.addMatrixExprParam(DataExpression.RAND_COLS,passedParamExprs.get(2).getExpr());
-
- if (unnamedParamCount >= 4)
- dataExpr.addMatrixExprParam(DataExpression.RAND_BY_ROW,passedParamExprs.get(3).getExpr());
-
- if (unnamedParamCount == 5)
- dataExpr.addMatrixExprParam(DataExpression.RAND_DIMNAMES,passedParamExprs.get(4).getExpr());
-
- if (unnamedParamCount > 5)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, at most 5 arguments supported (in order): data, rows, cols, byrow, dimname");
-
-
- } else {
- // handle first parameter, which is data and may be unnamed
- ParameterExpression firstParam = passedParamExprs.get(0);
- if (firstParam.getName() != null && !firstParam.getName().equals(DataExpression.RAND_DATA)){
- // throw exception -- must be filename as first parameter
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "matrix method must have data parameter as first parameter or unnamed parameter");
- } else {
- dataExpr.addMatrixExprParam(DataExpression.RAND_DATA, passedParamExprs.get(0).getExpr());
- }
-
- for (int i=1; i<passedParamExprs.size(); i++){
- if (passedParamExprs.get(i).getName() == null){
- // throw exception -- cannot mix named and unnamed parameters
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, cannot mix named and unnamed parameters, only data parameter can be unnammed");
- } else {
- dataExpr.addMatrixExprParam(passedParamExprs.get(i).getName(), passedParamExprs.get(i).getExpr());
- }
- }
- }
- dataExpr.setMatrixDefault();
- } // else if (functionName.equals("matrix")){
-
- if (dataExpr != null) {
- dataExpr.setAllPositions(filename, blp, bcp, elp, ecp);
- }
- return dataExpr;
-
- } // end method getBuiltinFunctionExpression
-
- public void addRandExprParam(String paramName, Expression paramValue) throws DMLParseException
- {
- // check name is valid
- boolean found = false;
- if (paramName != null ){
- for (String name : RAND_VALID_PARAM_NAMES){
- if (name.equals(paramName)) {
- found = true;
- break;
- }
- }
- }
- if (!found){
-
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "unexpected parameter \"" + paramName +
- "\". Legal parameters for Rand statement are "
- + "(capitalization-sensitive): " + RAND_ROWS
- + ", " + RAND_COLS + ", " + RAND_MIN + ", " + RAND_MAX
- + ", " + RAND_SPARSITY + ", " + RAND_SEED + ", " + RAND_PDF + ", " + RAND_LAMBDA);
- }
- if (getVarParam(paramName) != null){
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "attempted to add Rand statement parameter " + paramValue + " more than once");
- }
- // Process the case where user provides double values to rows or cols
- if (paramName.equals(RAND_ROWS) && paramValue instanceof DoubleIdentifier){
- paramValue = new IntIdentifier((long)((DoubleIdentifier)paramValue).getValue(),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- }
- else if (paramName.equals(RAND_COLS) && paramValue instanceof DoubleIdentifier){
- paramValue = new IntIdentifier((long)((DoubleIdentifier)paramValue).getValue(),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- }
-
- // add the parameter to expression list
- paramValue.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- addVarParam(paramName,paramValue);
-
- }
-
- public void addMatrixExprParam(String paramName, Expression paramValue) throws DMLParseException
- {
- // check name is valid
- boolean found = false;
- if (paramName != null ){
- for (String name : MATRIX_VALID_PARAM_NAMES){
- if (name.equals(paramName)) {
- found = true;
- }
- }
- }
-
- if (!found){
-
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "unexpected parameter \"" + paramName +
- "\". Legal parameters for matrix statement are "
- + "(capitalization-sensitive): " + RAND_DATA + ", " + RAND_ROWS
- + ", " + RAND_COLS + ", " + RAND_BY_ROW);
- }
- if (getVarParam(paramName) != null){
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "attempted to add matrix statement parameter " + paramValue + " more than once");
- }
- // Process the case where user provides double values to rows or cols
- if (paramName.equals(RAND_ROWS) && paramValue instanceof DoubleIdentifier){
- paramValue = new IntIdentifier((long)((DoubleIdentifier)paramValue).getValue(),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- }
- else if (paramName.equals(RAND_COLS) && paramValue instanceof DoubleIdentifier){
- paramValue = new IntIdentifier((long)((DoubleIdentifier)paramValue).getValue(),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- }
-
- // add the parameter to expression list
- paramValue.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- addVarParam(paramName,paramValue);
- }
-
- public DataExpression(DataOp op, HashMap<String,Expression> varParams,
- String filename, int blp, int bcp, int elp, int ecp) {
-
- _kind = Kind.DataOp;
- _opcode = op;
- _varParams = varParams;
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- HashMap<String,Expression> newVarParams = new HashMap<String,Expression>();
- for( Entry<String, Expression> e : _varParams.entrySet() ){
- String key = e.getKey();
- Expression newExpr = e.getValue().rewriteExpression(prefix);
- newVarParams.put(key, newExpr);
- }
- DataExpression retVal = new DataExpression(_opcode, newVarParams,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- retVal._strInit = this._strInit;
-
- return retVal;
- }
-
- /**
- * By default we use rowwise matrix reshape according to our internal dense/sparse matrix representations.
- * ByRow specifies both input and output orientation. Note that this is different from R, where inputs are
- * always read by-column and the default for byRow is by-column as well.
- */
- public void setMatrixDefault(){
- if (getVarParam(RAND_BY_ROW) == null)
- addVarParam(RAND_BY_ROW, new BooleanIdentifier(true,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
-
- public void setRandDefault(){
- if (getVarParam(RAND_ROWS)== null){
- IntIdentifier id = new IntIdentifier(1L,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_ROWS, id);
- }
- if (getVarParam(RAND_COLS)== null){
- IntIdentifier id = new IntIdentifier(1L,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_COLS, id);
- }
- if (getVarParam(RAND_MIN)== null){
- DoubleIdentifier id = new DoubleIdentifier(0.0,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_MIN, id);
- }
- if (getVarParam(RAND_MAX)== null){
- DoubleIdentifier id = new DoubleIdentifier(1.0,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_MAX, id);
- }
- if (getVarParam(RAND_SPARSITY)== null){
- DoubleIdentifier id = new DoubleIdentifier(1.0,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_SPARSITY, id);
- }
- if (getVarParam(RAND_SEED)== null){
- IntIdentifier id = new IntIdentifier(DataGenOp.UNSPECIFIED_SEED,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_SEED, id);
- }
- if (getVarParam(RAND_PDF)== null){
- StringIdentifier id = new StringIdentifier(RAND_PDF_UNIFORM,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_PDF, id);
- }
- if (getVarParam(RAND_LAMBDA)== null){
- DoubleIdentifier id = new DoubleIdentifier(1.0,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn());
- addVarParam(RAND_LAMBDA, id);
- }
- }
-
-
- public void setOpCode(DataOp op) {
- _opcode = op;
- }
-
- public DataOp getOpCode() {
- return _opcode;
- }
-
- public HashMap<String,Expression> getVarParams() {
- return _varParams;
- }
-
- public void setVarParams(HashMap<String, Expression> varParams) {
- _varParams = varParams;
- }
-
- public Expression getVarParam(String name) {
- return _varParams.get(name);
- }
-
-
-
- public void addVarParam(String name, Expression value){
- _varParams.put(name, value);
-
- // if required, initialize values
- setFilename(value.getFilename());
- if (getBeginLine() == 0) setBeginLine(value.getBeginLine());
- if (getBeginColumn() == 0) setBeginColumn(value.getBeginColumn());
- if (getEndLine() == 0) setEndLine(value.getEndLine());
- if (getEndColumn() == 0) setEndColumn(value.getEndColumn());
-
- }
-
- public void removeVarParam(String name) {
- _varParams.remove(name);
- }
-
- private String processInputFileName(HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
- throws LanguageException
- {
- String filename = null;
-
- Expression fileNameExpr = getVarParam(IO_FILENAME);
- if (fileNameExpr instanceof ConstIdentifier){
- return fileNameExpr.toString();
- }
- else if (fileNameExpr instanceof BinaryExpression){
- BinaryExpression expr = (BinaryExpression)fileNameExpr;
-
- if (expr.getKind()== Expression.Kind.BinaryOp){
- Expression.BinaryOp op = expr.getOpCode();
- switch (op){
- case PLUS:
- filename = "";
- filename = fileNameCat(expr, currConstVars, filename, conditional);
- // Since we have computed the value of filename, we update
- // varParams with a const string value
- StringIdentifier fileString = new StringIdentifier(filename,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- removeVarParam(IO_FILENAME);
- addVarParam(IO_FILENAME, fileString);
- break;
- default:
- raiseValidateError("for read method, parameter " + IO_FILENAME + " can only be const string concatenations. ", conditional);
- }
- }
- }
- else {
- raiseValidateError("for read method, parameter " + IO_FILENAME + " can only be a const string or const string concatenations. ", conditional);
- }
-
- return filename;
- }
-
- public static String getMTDFileName(String inputFileName) throws LanguageException {
- String mtdName = inputFileName + ".mtd";
-
- //validate read filename
- if( !LocalFileUtils.validateExternalFilename(mtdName, true) )
- throw new LanguageException("Invalid (non-trustworthy) hdfs read filename.");
-
- return mtdName;
- }
-
- /**
- * Validate parse tree : Process Data Expression in an assignment
- * statement
- *
- * @throws LanguageException
- * @throws ParseException
- * @throws IOException
- */
- @Override
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
- throws LanguageException
- {
- // validate all input parameters
- for ( Entry<String,Expression> e : getVarParams().entrySet() ) {
- String s = e.getKey();
- Expression inputParamExpr = e.getValue();
-
- if (inputParamExpr instanceof FunctionCallIdentifier) {
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false,LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- inputParamExpr.validateExpression(ids, currConstVars, conditional);
- if ( getVarParam(s).getOutput().getDataType() != DataType.SCALAR && !s.equals(RAND_DATA)) {
- raiseValidateError("Non-scalar data types are not supported for data expression.", conditional,LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- //general data expression constant propagation
- performConstantPropagationRand( currConstVars );
- performConstantPropagationReadWrite( currConstVars );
-
- // check if data parameter of matrix is scalar or matrix -- if scalar, use Rand instead
- Expression dataParam1 = getVarParam(RAND_DATA);
- if (dataParam1 == null && getOpCode().equals(DataOp.MATRIX)){
- raiseValidateError("for matrix, must defined data parameter", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if (dataParam1 != null && dataParam1.getOutput().getDataType() == DataType.SCALAR /*&& dataParam instanceof ConstIdentifier*/ ){
- //MB: note we should not check for const identifiers here, because otherwise all matrix constructors with
- //variable input are routed to a reshape operation (but it works only on matrices and hence, crashes)
-
- // replace DataOp MATRIX with RAND -- Rand handles matrix generation for Scalar values
- // replace data parameter with min / max within Rand case below
- this.setOpCode(DataOp.RAND);
- }
-
-
- // IMPORTANT: for each operation, one must handle unnamed parameters
-
- switch (this.getOpCode()) {
-
- case READ:
-
- if (getVarParam(DATATYPEPARAM) != null && !(getVarParam(DATATYPEPARAM) instanceof StringIdentifier)){
- raiseValidateError("for read statement, parameter " + DATATYPEPARAM + " can only be a string. " +
- "Valid values are: " + Statement.MATRIX_DATA_TYPE +", " + Statement.SCALAR_DATA_TYPE, conditional);
- }
-
- String dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString();
-
- // disallow certain parameters while reading a scalar
- if (dataTypeString != null && dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)){
- if ( getVarParam(READROWPARAM) != null
- || getVarParam(READCOLPARAM) != null
- || getVarParam(ROWBLOCKCOUNTPARAM) != null
- || getVarParam(COLUMNBLOCKCOUNTPARAM) != null
- || getVarParam(FORMAT_TYPE) != null
- || getVarParam(DELIM_DELIMITER) != null
- || getVarParam(DELIM_HAS_HEADER_ROW) != null
- || getVarParam(DELIM_FILL) != null
- || getVarParam(DELIM_FILL_VALUE) != null
- || getVarParam(DELIM_NA_STRINGS) != null
- )
- {
- raiseValidateError("Invalid parameters in read statement of a scalar: " +
- toString() + ". Only " + VALUETYPEPARAM + " is allowed.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- JSONObject configObject = null;
-
- // Process expressions in input filename
- String inputFileName = processInputFileName(currConstVars, conditional);
-
- // Obtain and validate metadata filename
- String mtdFileName = getMTDFileName(inputFileName);
-
-
- // track whether should attempt to read MTD file or not
- boolean shouldReadMTD = checkMetadata;
-
- // track whether format type has been inferred
- boolean inferredFormatType = false;
-
- // get format type string
- String formatTypeString = (getVarParam(FORMAT_TYPE) == null) ? null : getVarParam(FORMAT_TYPE).toString();
-
- // check if file is matrix market format
- if (formatTypeString == null && shouldReadMTD){
- boolean isMatrixMarketFormat = checkHasMatrixMarketFormat(inputFileName, mtdFileName, conditional);
- if (isMatrixMarketFormat){
-
- formatTypeString = FORMAT_TYPE_VALUE_MATRIXMARKET;
- addVarParam(FORMAT_TYPE,new StringIdentifier(FORMAT_TYPE_VALUE_MATRIXMARKET,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- inferredFormatType = true;
- shouldReadMTD = false;
- }
- }
-
- // check if file is delimited format
- if (formatTypeString == null) {
- boolean isDelimitedFormat = checkHasDelimitedFormat(inputFileName, conditional);
-
- if (isDelimitedFormat){
- addVarParam(FORMAT_TYPE,new StringIdentifier(FORMAT_TYPE_VALUE_CSV,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- formatTypeString = FORMAT_TYPE_VALUE_CSV;
- inferredFormatType = true;
- //shouldReadMTD = false;
- }
- }
-
-
- if (formatTypeString != null && formatTypeString.equalsIgnoreCase(FORMAT_TYPE_VALUE_MATRIXMARKET)){
- /*
- * handle MATRIXMARKET_FORMAT_TYPE format
- *
- * 1) only allow IO_FILENAME as ONLY valid parameter
- *
- * 2) open the file
- * A) verify header line (1st line) equals
- * B) read and discard comment lines
- * C) get size information from sizing info line --- M N L
- */
-
- for (String key : _varParams.keySet()){
- if ( !(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) ) ){
- raiseValidateError("Invalid parameters in readMM statement: " +
- toString() + ". Only " + IO_FILENAME + " is allowed.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
-
- // should NOT attempt to read MTD file for MatrixMarket format
- shouldReadMTD = false;
-
- // get metadata from MatrixMarket format file
- String[] headerLines = readMatrixMarketFile(inputFileName, conditional);
-
- // process 1st line of MatrixMarket format -- must be identical to legal header
- String legalHeaderMM = "%%MatrixMarket matrix coordinate real general";
-
- if (headerLines != null && headerLines.length >= 2){
- String firstLine = headerLines[0].trim();
- if (!firstLine.equals(legalHeaderMM)){
- raiseValidateError("Unsupported format in MatrixMarket file: " +
- headerLines[0] + ". Only supported format in MatrixMarket file has header line " + legalHeaderMM,
- conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // process 2nd line of MatrixMarket format -- must have size information
-
-
- String secondLine = headerLines[1];
- String[] sizeInfo = secondLine.trim().split("\\s+");
- if (sizeInfo.length != 3){
- raiseValidateError("Unsupported size line in MatrixMarket file: " +
- headerLines[1] + ". Only supported format in MatrixMarket file has size line: <NUM ROWS> <NUM COLS> <NUM NON-ZEROS>, where each value is an integer.", conditional);
- }
-
- long rowsCount = -1, colsCount = -1, nnzCount = -1;
- try {
- rowsCount = Long.parseLong(sizeInfo[0]);
- if (rowsCount < 1)
- throw new Exception("invalid rows count");
- addVarParam(READROWPARAM, new IntIdentifier(rowsCount,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- catch(Exception e){
- raiseValidateError("In MatrixMarket file " + getVarParam(IO_FILENAME)
- + " invalid row count " + sizeInfo[0] + " (must be long value >= 1). Sizing info line from file: " + headerLines[1], conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- try {
- colsCount = Long.parseLong(sizeInfo[1]);
- if (colsCount < 1)
- throw new Exception("invalid cols count");
- addVarParam(READCOLPARAM, new IntIdentifier(colsCount,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- catch(Exception e){
- raiseValidateError("In MatrixMarket file " + getVarParam(IO_FILENAME)
- + " invalid column count " + sizeInfo[1] + " (must be long value >= 1). Sizing info line from file: " + headerLines[1], conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- try {
- nnzCount = Long.parseLong(sizeInfo[2]);
- if (nnzCount < 1)
- throw new Exception("invalid nnz count");
- addVarParam("nnz", new IntIdentifier(nnzCount,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- catch(Exception e){
- raiseValidateError("In MatrixMarket file " + getVarParam(IO_FILENAME)
- + " invalid number non-zeros " + sizeInfo[2] + " (must be long value >= 1). Sizing info line from file: " + headerLines[1], conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- configObject = null;
-
- if (shouldReadMTD){
- configObject = readMetadataFile(mtdFileName, conditional);
-
- // if the MTD file exists, check the values specified in read statement match values in metadata MTD file
- if (configObject != null){
- parseMetaDataFileParameters(mtdFileName, configObject, conditional);
- }
- else {
- LOG.warn("Metadata file: " + new Path(mtdFileName) + " not provided");
- }
- }
-
- boolean isCSV = false;
- isCSV = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV));
- if (isCSV){
- // Handle delimited file format
- //
- // 1) only allow IO_FILENAME, _HEADER_ROW, FORMAT_DELIMITER, READROWPARAM, READCOLPARAM
- //
- // 2) open the file
- //
-
- // there should be no MTD file for delimited file format
- shouldReadMTD = true;
-
- // only allow IO_FILENAME, HAS_HEADER_ROW, FORMAT_DELIMITER, READROWPARAM, READCOLPARAM
- // as ONLY valid parameters
- if( !inferredFormatType ){
- for (String key : _varParams.keySet()){
- if (! (key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE)
- || key.equals(DELIM_HAS_HEADER_ROW) || key.equals(DELIM_DELIMITER)
- || key.equals(DELIM_FILL) || key.equals(DELIM_FILL_VALUE)
- || key.equals(READROWPARAM) || key.equals(READCOLPARAM)
- || key.equals(READNUMNONZEROPARAM) || key.equals(DATATYPEPARAM) || key.equals(VALUETYPEPARAM)
- )){
-
- String msg = "Only parameters allowed are: " + IO_FILENAME + ","
- + DELIM_HAS_HEADER_ROW + ","
- + DELIM_DELIMITER + ","
- + DELIM_FILL + ","
- + DELIM_FILL_VALUE + ","
- + READROWPARAM + ","
- + READCOLPARAM;
-
- raiseValidateError("Invalid parameter " + key + " in read.csv statement: " +
- toString() + ". " + msg, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- // DEFAULT for "sep" : ","
- if (getVarParam(DELIM_DELIMITER) == null){
- addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- else {
- if ( (getVarParam(DELIM_DELIMITER) instanceof ConstIdentifier)
- && (! (getVarParam(DELIM_DELIMITER) instanceof StringIdentifier)))
- {
- raiseValidateError("For delimited file '" + getVarParam(DELIM_DELIMITER)
- + "' must be a string value ", conditional);
- }
- }
-
- // DEFAULT for "default": 0
- if (getVarParam(DELIM_FILL_VALUE) == null){
- addVarParam(DELIM_FILL_VALUE, new DoubleIdentifier(DEFAULT_DELIM_FILL_VALUE,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- else {
- if ( (getVarParam(DELIM_FILL_VALUE) instanceof ConstIdentifier)
- && (! (getVarParam(DELIM_FILL_VALUE) instanceof IntIdentifier || getVarParam(DELIM_FILL_VALUE) instanceof DoubleIdentifier)))
- {
- raiseValidateError("For delimited file '" + getVarParam(DELIM_FILL_VALUE) + "' must be a numeric value ", conditional);
- }
- }
-
- // DEFAULT for "header": boolean false
- if (getVarParam(DELIM_HAS_HEADER_ROW) == null){
- addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- else {
- if ((getVarParam(DELIM_HAS_HEADER_ROW) instanceof ConstIdentifier)
- && (! (getVarParam(DELIM_HAS_HEADER_ROW) instanceof BooleanIdentifier)))
- {
- raiseValidateError("For delimited file '" + getVarParam(DELIM_HAS_HEADER_ROW) + "' must be a boolean value ", conditional);
- }
- }
-
- // DEFAULT for "fill": boolean false
- if (getVarParam(DELIM_FILL) == null){
- addVarParam(DELIM_FILL,new BooleanIdentifier(DEFAULT_DELIM_FILL,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- else {
-
- if ((getVarParam(DELIM_FILL) instanceof ConstIdentifier)
- && (! (getVarParam(DELIM_FILL) instanceof BooleanIdentifier)))
- {
- raiseValidateError("For delimited file '" + getVarParam(DELIM_FILL) + "' must be a boolean value ", conditional);
- }
- }
- }
- dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString();
-
- if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)
- || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) {
-
- boolean isMatrix = false;
- if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE))
- isMatrix = true;
-
- // set data type
- getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME);
-
- // set number non-zeros
- Expression ennz = this.getVarParam("nnz");
- long nnz = -1;
- if( ennz != null )
- {
- nnz = Long.valueOf(ennz.toString());
- getOutput().setNnz(nnz);
- }
-
- // Following dimension checks must be done when data type = MATRIX_DATA_TYPE
- // initialize size of target data identifier to UNKNOWN
- getOutput().setDimensions(-1, -1);
-
- if ( !isCSV && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null)){
- raiseValidateError("Missing or incomplete dimension information in read statement: " + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if (getVarParam(READROWPARAM) instanceof ConstIdentifier && getVarParam(READCOLPARAM) instanceof ConstIdentifier) {
-
- // these are strings that are long values
- Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString());
- Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString());
-
- if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && REJECT_READ_UNKNOWN_SIZE )
- {
- raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- // set dim1 and dim2 values
- if (dim1 != null && dim2 != null){
- getOutput().setDimensions(dim1, dim2);
- } else if (!isCSV && ((dim1 != null) || (dim2 != null))) {
- raiseValidateError("Partial dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- // Table must be in CSV format
- if ( !isMatrix && !getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("csv") ) {
- raiseValidateError("Input data of type 'table' must be in CSV format. Invalid format '" + getVarParam(FORMAT_TYPE)+ "' in statement: " + this.toString(), conditional);
- }
-
- // initialize block dimensions to UNKNOWN
- getOutput().setBlockDimensions(-1, -1);
-
- // find "format": 1=text, 2=binary
- int format = 1; // default is "text"
- String fmt = (getVarParam(FORMAT_TYPE) == null ? null : getVarParam(FORMAT_TYPE).toString());
-
- if (fmt == null || fmt.equalsIgnoreCase("text")){
- getOutput().setFormatType(FormatType.TEXT);
- format = 1;
- } else if ( fmt.equalsIgnoreCase("binary") ) {
- getOutput().setFormatType(FormatType.BINARY);
- format = 2;
- } else if ( fmt.equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV))
- {
- getOutput().setFormatType(FormatType.CSV);
- format = 1;
- }
- else if ( fmt.equalsIgnoreCase(FORMAT_TYPE_VALUE_MATRIXMARKET) )
- {
- getOutput().setFormatType(FormatType.MM);
- format = 1;
- } else {
- raiseValidateError("Invalid format '" + fmt+ "' in statement: " + this.toString(), conditional);
- }
-
- if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) {
-
- Long rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? null : Long.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString());
- Long columnBlockCount = (getVarParam(COLUMNBLOCKCOUNTPARAM) == null) ? null : Long.valueOf(getVarParam(COLUMNBLOCKCOUNTPARAM).toString());
-
- if ((rowBlockCount != null) && (columnBlockCount != null)) {
- getOutput().setBlockDimensions(rowBlockCount, columnBlockCount);
- } else if ((rowBlockCount != null) || (columnBlockCount != null)) {
- raiseValidateError("Partial block dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- } else {
- getOutput().setBlockDimensions(-1, -1);
- }
- }
-
- // block dimensions must be -1x-1 when format="text"
- // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary"
- // because we automatically introduce reblocks if blocksizes don't match
- if ( ( (format == 1 || !isMatrix)
- && (getOutput().getRowsInBlock() != -1 || getOutput().getColumnsInBlock() != -1)
- ) ){
- raiseValidateError("Invalid block dimensions (" + getOutput().getRowsInBlock() + "," + getOutput().getColumnsInBlock() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional);
- }
-
- }
- else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) {
- getOutput().setDataType(DataType.SCALAR);
- getOutput().setNnz(-1L);
- }
-
- else{
- raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // handle value type parameter
- if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){
- raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " +
- "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional);
- }
- // Identify the value type (used only for read method)
- String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString();
- if (valueTypeString != null) {
- if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) {
- getOutput().setValueType(ValueType.DOUBLE);
- } else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) {
- getOutput().setValueType(ValueType.STRING);
- } else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) {
- getOutput().setValueType(ValueType.INT);
- } else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) {
- getOutput().setValueType(ValueType.BOOLEAN);
- } else {
- raiseValidateError("Unknown Value Type " + valueTypeString
- + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional);
- }
- } else {
- getOutput().setValueType(ValueType.DOUBLE);
- }
-
- break;
-
- case WRITE:
-
- // for delimited format, if no delimiter specified THEN set default ","
- if (getVarParam(FORMAT_TYPE) == null || getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV)){
- if (getVarParam(DELIM_DELIMITER) == null){
- addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- if (getVarParam(DELIM_HAS_HEADER_ROW) == null){
- addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- if (getVarParam(DELIM_SPARSE) == null){
- addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getBeginLine(), this.getBeginColumn()));
- }
- }
-
- /* NOTE MB: disabled filename concatenation because we now support dynamic rewrite
- if (getVarParam(IO_FILENAME) instanceof BinaryExpression){
- BinaryExpression expr = (BinaryExpression)getVarParam(IO_FILENAME);
-
- if (expr.getKind()== Expression.Kind.BinaryOp){
- Expression.BinaryOp op = expr.getOpCode();
- switch (op){
- case PLUS:
- mtdFileName = "";
- mtdFileName = fileNameCat(expr, currConstVars, mtdFileName);
- // Since we have computed the value of filename, we update
- // varParams with a const string value
- StringIdentifier fileString = new StringIdentifier(mtdFileName,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- removeVarParam(IO_FILENAME);
- addVarParam(IO_FILENAME, fileString);
-
- break;
- default:
- raiseValidateError("for OutputStatement, parameter " + IO_FILENAME
- + " can only be a const string or const string concatenations. ",
- conditional);
- }
- }
- }*/
-
- //validate read filename
- String fnameWrite = getVarParam(IO_FILENAME).toString();
- if( !LocalFileUtils.validateExternalFilename(fnameWrite, true) ) //always unconditional
- raiseValidateError("Invalid (non-trustworthy) hdfs write filename.", false);
-
-
- if (getVarParam(FORMAT_TYPE) == null || getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("text"))
- getOutput().setBlockDimensions(-1, -1);
- else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("binary"))
- getOutput().setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_MATRIXMARKET) || (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV)))
- getOutput().setBlockDimensions(-1, -1);
-
- else{
- raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + this.toString(), conditional);
- }
- break;
-
- case RAND:
-
- Expression dataParam = getVarParam(RAND_DATA);
-
- if( dataParam != null )
- {
- // handle input variable (matrix/scalar)
- if( dataParam instanceof DataIdentifier )
- {
- addVarParam(RAND_MIN, dataParam);
- addVarParam(RAND_MAX, dataParam);
- }
- // handle integer constant
- else if (dataParam instanceof IntIdentifier) {
- long roundedValue = ((IntIdentifier)dataParam).getValue();
- Expression minExpr = new DoubleIdentifier(roundedValue, getFilename(),
- getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn());
- addVarParam(RAND_MIN, minExpr);
- addVarParam(RAND_MAX, minExpr);
- }
- // handle double constant
- else if (dataParam instanceof DoubleIdentifier) {
- double roundedValue = ((DoubleIdentifier)dataParam).getValue();
- Expression minExpr = new DoubleIdentifier(roundedValue, getFilename(),
- getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn());
- addVarParam(RAND_MIN, minExpr);
- addVarParam(RAND_MAX, minExpr);
- }
- // handle string constant (string init)
- else if (dataParam instanceof StringIdentifier) {
- String data = ((StringIdentifier)dataParam).getValue();
- Expression minExpr = new StringIdentifier(data, getFilename(),
- getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn());
- addVarParam(RAND_MIN, minExpr);
- addVarParam(RAND_MAX, minExpr);
- _strInit = true;
- }
- else {
- // handle general expression
- dataParam.validateExpression(ids, currConstVars, conditional);
- addVarParam(RAND_MIN, dataParam);
- addVarParam(RAND_MAX, dataParam);
- }
-
- removeVarParam(RAND_DATA);
- removeVarParam(RAND_BY_ROW);
- this.setRandDefault();
- }
-
- //check valid parameters
- for( String key : _varParams.keySet() ) {
- boolean found = false;
- for (String name : RAND_VALID_PARAM_NAMES){
- found |= name.equals(key);
- }
- if (!found){
- raiseValidateError("unexpected parameter \"" + key +
- "\". Legal parameters for Rand statement are "
- + "(capitalization-sensitive): " + RAND_ROWS
- + ", " + RAND_COLS + ", " + RAND_MIN + ", " + RAND_MAX
- + ", " + RAND_SPARSITY + ", " + RAND_SEED + ", " + RAND_PDF + ", " + RAND_LAMBDA, conditional);
- }
- }
-
- //parameters w/ support for variable inputs
- if (getVarParam(RAND_ROWS) instanceof StringIdentifier || getVarParam(RAND_ROWS) instanceof BooleanIdentifier){
- raiseValidateError("for Rand statement " + RAND_ROWS + " has incorrect value type", conditional);
- }
-
- if (getVarParam(RAND_COLS) instanceof StringIdentifier || getVarParam(RAND_COLS) instanceof BooleanIdentifier){
- raiseValidateError("for Rand statement " + RAND_COLS + " has incorrect value type", conditional);
- }
-
- if (getVarParam(RAND_SEED) instanceof StringIdentifier || getVarParam(RAND_SEED) instanceof BooleanIdentifier) {
- raiseValidateError("for Rand statement " + RAND_SEED + " has incorrect value type", conditional);
- }
-
- if ((getVarParam(RAND_MAX) instanceof StringIdentifier && !_strInit) || getVarParam(RAND_MAX) instanceof BooleanIdentifier) {
- raiseValidateError("for Rand statement " + RAND_MAX + " has incorrect value type", conditional);
- }
-
- if ((getVarParam(RAND_MIN) instanceof StringIdentifier && !_strInit) || getVarParam(RAND_MIN) instanceof BooleanIdentifier) {
- raiseValidateError("for Rand statement " + RAND_MIN + " has incorrect value type", conditional);
- }
-
- //parameters w/o support for variable inputs (requires double/int or string constants)
- if (!(getVarParam(RAND_SPARSITY) instanceof DoubleIdentifier || getVarParam(RAND_SPARSITY) instanceof IntIdentifier)) {
- raiseValidateError("for Rand statement " + RAND_SPARSITY + " has incorrect value type", conditional);
- }
-
- if (!(getVarParam(RAND_PDF) instanceof StringIdentifier)) {
- raiseValidateError("for Rand statement " + RAND_PDF + " has incorrect value type", conditional);
- }
-
- Expression lambda = getVarParam(RAND_LAMBDA);
- if (!( (lambda instanceof DataIdentifier
- || lambda instanceof ConstIdentifier)
- && (lambda.getOutput().getValueType() == ValueType.DOUBLE
- || lambda.getOutput().getValueType() == ValueType.INT) )) {
- raiseValidateError("for Rand statement " + RAND_LAMBDA + " has incorrect data type", conditional);
- }
-
- long rowsLong = -1L, colsLong = -1L;
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE ROWS
- ///////////////////////////////////////////////////////////////////
- Expression rowsExpr = getVarParam(RAND_ROWS);
- if (rowsExpr instanceof IntIdentifier) {
- if (((IntIdentifier)rowsExpr).getValue() >= 1 ) {
- rowsLong = ((IntIdentifier)rowsExpr).getValue();
- }
- else {
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + ((IntIdentifier)rowsExpr).getValue(), conditional);
- }
- }
- else if (rowsExpr instanceof DoubleIdentifier) {
- if (((DoubleIdentifier)rowsExpr).getValue() >= 1 ) {
- rowsLong = UtilFunctions.toLong(Math.floor(((DoubleIdentifier)rowsExpr).getValue()));
- }
- else {
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + rowsExpr.toString(), conditional);
- }
- }
- else if (rowsExpr instanceof DataIdentifier && !(rowsExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)rowsExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // check rows is >= 1 --- throw exception
- if (((IntIdentifier)constValue).getValue() < 1){
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update row expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- rowsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_ROWS, rowsExpr);
- rowsLong = roundedValue;
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- if (((DoubleIdentifier)constValue).getValue() < 1.0){
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update row expr with new IntIdentifier (rounded down)
- long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue();
- rowsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_ROWS, rowsExpr);
- rowsLong = roundedValue;
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- rowsExpr.validateExpression(ids, currConstVars, conditional);
- }
- }
- else {
- // handle general expression
- rowsExpr.validateExpression(ids, currConstVars, conditional);
- }
-
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE COLUMNS
- ///////////////////////////////////////////////////////////////////
-
- Expression colsExpr = getVarParam(RAND_COLS);
- if (colsExpr instanceof IntIdentifier) {
- if (((IntIdentifier)colsExpr).getValue() >= 1 ) {
- colsLong = ((IntIdentifier)colsExpr).getValue();
- }
- else {
- raiseValidateError("In rand statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional);
- }
- }
- else if (colsExpr instanceof DoubleIdentifier) {
- if (((DoubleIdentifier)colsExpr).getValue() >= 1 ) {
- colsLong = Double.valueOf((Math.floor(((DoubleIdentifier)colsExpr).getValue()))).longValue();
- }
- else {
- raiseValidateError("In rand statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional);
- }
- }
- else if (colsExpr instanceof DataIdentifier && !(colsExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)colsExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // check cols is >= 1 --- throw exception
- if (((IntIdentifier)constValue).getValue() < 1){
- raiseValidateError("In rand statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update col expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- colsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_COLS, colsExpr);
- colsLong = roundedValue;
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- if (((DoubleIdentifier)constValue).getValue() < 1){
- raiseValidateError("In rand statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update col expr with new IntIdentifier (rounded down)
- long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue();
- colsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_COLS, colsExpr);
- colsLong = roundedValue;
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In rand statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- colsExpr.validateExpression(ids, currConstVars, conditional);
- }
-
- }
- else {
- // handle general expression
- colsExpr.validateExpression(ids, currConstVars, conditional);
- }
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE MIN
- ///////////////////////////////////////////////////////////////////
- Expression minExpr = getVarParam(RAND_MIN);
-
- // perform constant propogation
- if (minExpr instanceof DataIdentifier && !(minExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)minExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // update min expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- minExpr = new DoubleIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_MIN, minExpr);
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- // update col expr with new IntIdentifier (rounded down)
- double roundedValue = ((DoubleIdentifier)constValue).getValue();
- minExpr = new DoubleIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_MIN, minExpr);
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In rand statement, can only assign min a numerical " +
- "value -- attempted to assign: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- minExpr.validateExpression(ids, currConstVars, conditional);
- }
-
- }
- else {
- // handle general expression
- minExpr.validateExpression(ids, currConstVars, conditional);
- }
-
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE MAX
- ///////////////////////////////////////////////////////////////////
- Expression maxExpr = getVarParam(RAND_MAX);
-
- // perform constant propogation
- if (maxExpr instanceof DataIdentifier && !(maxExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)maxExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // update min expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- maxExpr = new DoubleIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_MAX, maxExpr);
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- // update col expr with new IntIdentifier (rounded down)
- double roundedValue = ((DoubleIdentifier)constValue).getValue();
- maxExpr = new DoubleIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_MAX, maxExpr);
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In rand statement, can only assign max a numerical " +
- "value -- attempted to assign: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- maxExpr.validateExpression(ids, currConstVars, conditional);
- }
- }
- else {
- // handle general expression
- maxExpr.validateExpression(ids, currConstVars, conditional);
- }
-
- getOutput().setFormatType(FormatType.BINARY);
- getOutput().setDataType(DataType.MATRIX);
- getOutput().setValueType(ValueType.DOUBLE);
- getOutput().setDimensions(rowsLong, colsLong);
-
- if (getOutput() instanceof IndexedIdentifier){
- // process the "target" being indexed
- DataIdentifier targetAsSeen = ids.get(((DataIdentifier)getOutput()).getName());
- if (targetAsSeen == null){
- raiseValidateError("cannot assign value to indexed identifier " + ((DataIdentifier)getOutput()).getName() + " without first initializing " + ((DataIdentifier)getOutput()).getName(), conditional);
- }
- //_output.setProperties(targetAsSeen);
- ((IndexedIdentifier) getOutput()).setOriginalDimensions(targetAsSeen.getDim1(), targetAsSeen.getDim2());
- //((IndexedIdentifier) getOutput()).setOriginalDimensions(getOutput().getDim1(), getOutput().getDim2());
- }
- //getOutput().computeDataType();
-
- if (getOutput() instanceof IndexedIdentifier){
- LOG.warn(this.printWarningLocation() + "Output for Rand Statement may have incorrect size information");
- }
-
- break;
-
- case MATRIX:
-
- //handle default and input arguments
- setMatrixDefault();
- for( String key : _varParams.keySet() )
- {
- boolean found = false;
- for (String name : MATRIX_VALID_PARAM_NAMES) {
- found |= name.equals(key);
- }
- if( !found ) {
- raiseValidateError("unexpected parameter \"" + key + "\". "
- + "Legal parameters for matrix statement are (case-sensitive): "
- + RAND_DATA + ", " + RAND_ROWS + ", " + RAND_COLS + ", " + RAND_BY_ROW, conditional);
- }
- }
-
- //validate correct value types
- if (getVarParam(RAND_DATA) != null && (getVarParam(RAND_DATA) instanceof BooleanIdentifier)){
- raiseValidateError("for matrix statement " + RAND_DATA + " has incorrect value type", conditional);
- }
- if (getVarParam(RAND_ROWS) != null && (getVarParam(RAND_ROWS) instanceof StringIdentifier || getVarParam(RAND_ROWS) instanceof BooleanIdentifier)){
- raiseValidateError("for matrix statement " + RAND_ROWS + " has incorrect value type", conditional);
- }
- if (getVarParam(RAND_COLS) != null && (getVarParam(RAND_COLS) instanceof StringIdentifier || getVarParam(RAND_COLS) instanceof BooleanIdentifier)){
- raiseValidateError("for matrix statement " + RAND_COLS + " has incorrect value type", conditional);
- }
- if ( !(getVarParam(RAND_BY_ROW) instanceof BooleanIdentifier)) {
- raiseValidateError("for matrix statement " + RAND_BY_ROW + " has incorrect value type", conditional);
- }
-
- //validate general data expression
- getVarParam(RAND_DATA).validateExpression(ids, currConstVars, conditional);
-
- rowsLong = -1L;
- colsLong = -1L;
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE ROWS
- ///////////////////////////////////////////////////////////////////
- rowsExpr = getVarParam(RAND_ROWS);
- if (rowsExpr != null){
- if (rowsExpr instanceof IntIdentifier) {
- if (((IntIdentifier)rowsExpr).getValue() >= 1 ) {
- rowsLong = ((IntIdentifier)rowsExpr).getValue();
- }
- else {
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + ((IntIdentifier)rowsExpr).getValue(), conditional);
- }
- }
- else if (rowsExpr instanceof DoubleIdentifier) {
- if (((DoubleIdentifier)rowsExpr).getValue() >= 1 ) {
- rowsLong = Double.valueOf((Math.floor(((DoubleIdentifier)rowsExpr).getValue()))).longValue();
- }
- else {
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + rowsExpr.toString(), conditional);
- }
- }
- else if (rowsExpr instanceof DataIdentifier && !(rowsExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)rowsExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // check rows is >= 1 --- throw exception
- if (((IntIdentifier)constValue).getValue() < 1){
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update row expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- rowsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_ROWS, rowsExpr);
- rowsLong = roundedValue;
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- if (((DoubleIdentifier)constValue).getValue() < 1.0){
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- // update row expr with new IntIdentifier (rounded down)
- long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue();
- rowsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_ROWS, rowsExpr);
- rowsLong = roundedValue;
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- rowsExpr.validateExpression(ids, currConstVars, conditional);
- }
- }
- else {
- // handle general expression
- rowsExpr.validateExpression(ids, currConstVars, conditional);
- }
- }
-
- ///////////////////////////////////////////////////////////////////
- // HANDLE COLUMNS
- ///////////////////////////////////////////////////////////////////
-
- colsExpr = getVarParam(RAND_COLS);
- if (colsExpr != null){
- if (colsExpr instanceof IntIdentifier) {
- if (((IntIdentifier)colsExpr).getValue() >= 1 ) {
- colsLong = ((IntIdentifier)colsExpr).getValue();
- }
- else {
- raiseValidateError("In matrix statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional);
- }
- }
- else if (colsExpr instanceof DoubleIdentifier) {
- if (((DoubleIdentifier)colsExpr).getValue() >= 1 ) {
- colsLong = Double.valueOf((Math.floor(((DoubleIdentifier)colsExpr).getValue()))).longValue();
- }
- else {
- raiseValidateError("In matrix statement, can only assign rows a long " +
- "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional);
- }
- }
- else if (colsExpr instanceof DataIdentifier && !(colsExpr instanceof IndexedIdentifier)) {
-
- // check if the DataIdentifier variable is a ConstIdentifier
- String identifierName = ((DataIdentifier)colsExpr).getName();
- if (currConstVars.containsKey(identifierName)){
-
- // handle int constant
- ConstIdentifier constValue = currConstVars.get(identifierName);
- if (constValue instanceof IntIdentifier){
-
- // check cols is >= 1 --- throw exception
- if (((IntIdentifier)constValue).getValue() < 1){
- raiseValidateError("In matrix statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: "
- + constValue.toString(), conditional);
- }
- // update col expr with new IntIdentifier
- long roundedValue = ((IntIdentifier)constValue).getValue();
- colsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_COLS, colsExpr);
- colsLong = roundedValue;
- }
- // handle double constant
- else if (constValue instanceof DoubleIdentifier){
-
- if (((DoubleIdentifier)constValue).getValue() < 1){
- raiseValidateError("In matrix statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: "
- + constValue.toString(), conditional);
- }
- // update col expr with new IntIdentifier (rounded down)
- long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue();
- colsExpr = new IntIdentifier(roundedValue, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- addVarParam(RAND_COLS, colsExpr);
- colsLong = roundedValue;
-
- }
- else {
- // exception -- rows must be integer or double constant
- raiseValidateError("In matrix statement, can only assign cols a long " +
- "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional);
- }
- }
- else {
- // handle general expression
- colsExpr.validateExpression(ids, currConstVars, conditional);
- }
-
- }
- else {
- // handle general expression
- colsExpr.validateExpression(ids, currConstVars, conditional);
- }
- }
- getOutput().setFormatType(FormatType.BINARY);
- getOutput().setDataType(DataType.MATRIX);
- getOutput().setValueType(ValueType.DOUBLE);
- getOutput().setDimensions(rowsLong, colsLong);
-
- if (getOutput() instanceof IndexedIdentifier){
- ((IndexedIdentifier) getOutput()).setOriginalDimensions(getOutput().getDim1(), getOutput().getDim2());
- }
- //getOutput().computeDataType();
-
- if (getOutput() instanceof IndexedIdentifier){
- LOG.warn(this.printWarningLocation() + "Output for matrix Statement may have incorrect size information");
- }
-
- break;
-
-
- default:
- raiseValidateError("Unsupported Data expression"+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS); //always unconditional
- }
- return;
- }
-
- /**
- *
- * @param currConstVars
- */
- private void performConstantPropagationRand( HashMap<String, ConstIdentifier> currConstVars )
- {
- //here, we propagate constants for all rand parameters that are required during validate.
- String[] paramNamesForEval = new String[]{RAND_DATA, RAND_SPARSITY, RAND_MIN, RAND_MAX};
-
- //replace data identifiers with const identifiers
- performConstantPropagation(currConstVars, paramNamesForEval);
- }
-
- /**
- *
- * @param currConstVars
- */
- private void performConstantPropagationReadWrite( HashMap<String, ConstIdentifier> currConstVars )
- {
- //here, we propagate constants for all read/write parameters that are required during validate.
- String[] paramNamesForEval = new String[]{FORMAT_TYPE, IO_FILENAME, READROWPARAM, READCOLPARAM, READNUMNONZEROPARAM};
-
- //replace data identifiers with const identifiers
- performConstantPropagation(currConstVars, paramNamesForEval);
- }
-
- /**
- *
- * @param currConstVars
- * @param paramNames
- */
- private void performConstantPropagation( HashMap<String, ConstIdentifier> currConstVars, String[] paramNames )
- {
- for( String paramName : paramNames )
- {
- Expression paramExp = getVarParam(paramName);
- if ( paramExp != null && paramExp instanceof DataIdentifier && !(paramExp instanceof IndexedIdentifier)
- && currConstVars.containsKey(((DataIdentifier) paramExp).getName()))
- {
- addVarParam(paramName, currConstVars.get(((DataIdentifier)paramExp).getName()));
- }
- }
- }
-
-
- private String fileNameCat(BinaryExpression expr, HashMap<String, ConstIdentifier> currConstVars, String filename, boolean conditional)
- throws LanguageException
- {
- // Processing the left node first
- if (expr.getLeft() instanceof BinaryExpression
- && ((BinaryExpression)expr.getLeft()).getKind()== BinaryExpression.Kind.BinaryOp
- && ((BinaryExpression)expr.getLeft()).getOpCode() == BinaryOp.PLUS){
- filename = fileNameCat((BinaryExpression)expr.getLeft(), currConstVars, filename, conditional)+ filename;
- }
- else if (expr.getLeft() instanceof ConstIdentifier){
- filename = ((ConstIdentifier)expr.getLeft()).toString()+ filename;
- }
- else if (expr.getLeft() instanceof DataIdentifier
- && ((DataIdentifier)expr.getLeft()).getDataType() == Expression.DataType.SCALAR
- && ((DataIdentifier)expr.getLeft()).getKind() == Expression.Kind.Data){
- //&& ((DataIdentifier)expr.getLeft()).getValueType() == Expression.ValueType.STRING){
- String name = ((DataIdentifier)expr.getLeft()).getName();
- filename = ((StringIdentifier)currConstVars.get(name)).getValue() + filename;
- }
- else {
- raiseValidateError("Parameter " + IO_FILENAME + " only supports a const string or const string concatenations.", conditional);
- }
- // Now process the right node
- if (expr.getRight()instanceof BinaryExpression
- && ((BinaryExpression)expr.getRight()).getKind()== BinaryExpression.Kind.BinaryOp
- && ((BinaryExpression)expr.getRight()).getOpCode() == BinaryOp.PLUS){
- filename = filename + fileNameCat((BinaryExpression)expr.getRight(), currConstVars, filename, conditional);
- }
- // DRB: CHANGE
- else if (expr.getRight() instanceof ConstIdentifier){
- filename = filename + ((ConstIdentifier)expr.getRight()).toString();
- }
- else if (expr.getRight() instanceof DataIdentifier
- && ((DataIdentifier)expr.getRight()).getDataType() == Expression.DataType.SCALAR
- && ((DataIdentifier)expr.getRight()).getKind() == Expression.Kind.Data
- && ((DataIdentifier)expr.getRight()).getValueType() == Expression.ValueType.STRING){
- String name = ((DataIdentifier)expr.getRight()).getName();
- filename = filename + ((StringIdentifier)currConstVars.get(name)).getValue();
- }
- else {
- raiseValidateError("Parameter " + IO_FILENAME + " only supports a const string or const string concatenations.", conditional);
- }
- return filename;
-
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(_opcode.toString());
- sb.append("(");
-
- for(Entry<String,Expression> e : _varParams.entrySet()) {
- String key = e.getKey();
- Expression expr = e.getValue();
- sb.append(",");
- sb.append(key);
- sb.append("=");
- sb.append(expr);
- }
- sb.append(" )");
- return sb.toString();
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- for( Expression expr : _varParams.values() ) {
- result.addVariables ( expr.variablesRead() );
- }
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- for( Expression expr : _varParams.values() ) {
- result.addVariables ( expr.variablesUpdated() );
- }
- result.addVariable(((DataIdentifier)this.getOutput()).getName(), (DataIdentifier)this.getOutput());
- return result;
- }
-
- @SuppressWarnings("unchecked")
- private void parseMetaDataFileParameters(String mtdFileName, JSONObject configObject, boolean conditional)
- throws LanguageException
- {
- for( Object obj : configObject.entrySet() ){
- Entry<Object,Object> e = (Entry<Object, Object>) obj;
- Object key = e.getKey();
- Object val = e.getValue();
-
- boolean isValidName = false;
- for (String paramName : READ_VALID_MTD_PARAM_NAMES){
- if (paramName.equals(key))
- isValidName = true;
- }
-
- if (!isValidName){ //wrong parameters always rejected
- raiseValidateError("MTD file " + mtdFileName + " contains invalid parameter name: " + key, false);
- }
-
- // if the read method parameter is a constant, then verify value matches MTD metadata file
- if (getVarParam(key.toString()) != null && (getVarParam(key.toString()) instanceof ConstIdentifier)
- && !getVarParam(key.toString()).toString().equalsIgnoreCase(val.toString()) )
- {
- raiseValidateError("parameter " + key.toString() + " has conflicting values in read statement definition and metadata. " +
- "Config file value: " + val.toString() + " from MTD file. Read statement value: " + getVarParam(key.toString()), conditional);
- }
- else
- {
- // if the read method does not specify parameter value, then add MTD metadata file value to parameter list
- if (getVarParam(key.toString()) == null){
- if ( !key.toString().equalsIgnoreCase(DESCRIPTIONPARAM) ) {
- StringIdentifier strId = new StringIdentifier(val.toString(),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
-
- if ( key.toString().equalsIgnoreCase(DELIM_HAS_HEADER_ROW)
- || key.toString().equalsIgnoreCase(DELIM_FILL)
- || key.toString().equalsIgnoreCase(DELIM_SPARSE)
- ) {
- // parse these parameters as boolean values
- BooleanIdentifier boolId = null;
- if ( strId.toString().equalsIgnoreCase("true") ) {
- boolId = new BooleanIdentifier(true, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- }
- else if ( strId.toString().equalsIgnoreCase("false") ) {
- boolId = new BooleanIdentifier(false, this.getFilename(),
- this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- }
- else {
- raiseValidateError("Invalid value provided for '" + DELIM_HAS_HEADER_ROW + "' in metadata file '" + mtdFileName + "'. "
- + "Must be either TRUE or FALSE.", conditional);
- }
- removeVarParam(key.toString());
- addVarParam(key.toString(), boolId);
- }
- else if ( key.toString().equalsIgnoreCase(DELIM_FILL_VALUE)) {
- // parse these parameters as numeric values
- DoubleIdentifier doubleId = new DoubleIdentifier( Double.parseDouble(strId.toString()),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- removeVarParam(key.toString());
- addVarParam(key.toString(), doubleId);
- }
- else if (key.toString().equalsIgnoreCase(DELIM_NA_STRINGS)) {
- String naStrings = null;
- if ( val instanceof String) {
- naStrings = val.toString();
- }
- else {
- StringBuilder sb = new StringBuilder();
- JSONArray valarr = (JSONArray)val;
- for(int naid=0; naid < valarr.size(); naid++ ) {
- sb.append( (String) valarr.get(naid) );
- if ( naid < valarr.size()-1)
- sb.append( DELIM_NA_STRING_SEP );
- }
- naStrings = sb.toString();
- }
- StringIdentifier sid = new StringIdentifier( naStrings,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- removeVarParam(key.toString());
- addVarParam(key.toString(), sid);
- }
- else {
- // by default, treat a parameter as a string
- addVarParam(key.toString(), strId);
- }
- }
- }
- }
- }
- }
-
- /**
- *
- * @param filename
- * @return
- * @throws LanguageException
- */
- public JSONObject readMetadataFile(String filename, boolean conditional)
- throws LanguageException
- {
- JSONObject retVal = null;
- boolean exists = false;
- FileSystem fs = null;
-
- try {
- fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- } catch (Exception e){
- raiseValidateError("could not read the configuration file: "+e.getMessage(), false);
- }
-
- Path pt = new Path(filename);
- try {
- if (fs.exists(pt)){
- exists = true;
- }
- } catch (Exception e){
- exists = false;
- }
-
- boolean isDirBoolean = false;
- try {
- if (exists && fs.getFileStatus(pt).isDirectory())
- isDirBoolean = true;
- else
- isDirBoolean = false;
- }
- catch(Exception e){
- raiseValidateError("error validing whether path " + pt.toString() + " is directory or not: "+e.getMessage(), conditional);
- }
-
- // CASE: filename is a directory -- process as a directory
- if (exists && isDirBoolean){
-
- // read directory contents
- retVal = new JSONObject();
-
- FileStatus[] stats = null;
-
- try {
- stats = fs.listStatus(pt);
- }
- catch (Exception e){
- raiseValidateError("for MTD file in directory, error reading directory with MTD file " + pt.toString() + ": " + e.getMessage(), conditional);
- }
-
- for(FileStatus stat : stats){
- Path childPath = stat.getPath(); // gives directory name
- if (childPath.getName().startsWith("part")){
-
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(fs.open(childPath)));
- }
- catch(Exception e){
- raiseValidateError("for MTD file in directory, error reading part of MTD file with path " + childPath.toString() + ": " + e.getMessage(), conditional);
- }
-
- JSONObject childObj = null;
- try {
- childObj = JSONHelper.parse(br);
- }
- catch(Exception e){
- raiseValidateError("for MTD file in directory, error parsing part of MTD file with path " + childPath.toString() + ": " + e.getMessage(), conditional);
- }
-
- for( Object obj : childObj.entrySet() ){
- @SuppressWarnings("unchecked")
- Entry<Object,Object> e = (Entry<Object, Object>) obj;
- Object key = e.getKey();
- Object val = e.getValue();
- retVal.put(key, val);
- }
- }
- } // end for
- }
-
- // CASE: filename points to a file
- else if (exists){
-
- BufferedReader br = null;
-
- // try reading MTD file
- try {
- br=new BufferedReader(new InputStreamReader(fs.open(pt)));
- } catch (Exception e){
- raiseValidateError("error reading MTD file with path " + pt.toString() + ": " + e.getMessage(), conditional);
- }
-
- // try parsing MTD file
- try {
- retVal = JSONHelper.parse(br);
- } catch (Exception e){
- raiseValidateError("error parsing MTD file with path " + pt.toString() + ": " + e.getMessage(), conditional);
- }
- }
-
- return retVal;
- }
-
- public String[] readMatrixMarketFile(String filename, boolean conditional)
- throws LanguageException
- {
- String[] retVal = new String[2];
- retVal[0] = new String("");
- retVal[1] = new String("");
- boolean exists = false;
-
- try
- {
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path pt = new Path(filename);
- if (fs.exists(pt)){
- exists = true;
- }
-
- boolean getFileStatusIsDir = fs.getFileStatus(pt).isDirectory();
-
- if (exists && getFileStatusIsDir){
- raiseValidateError("MatrixMarket files as directories not supported", conditional);
- }
- else if (exists) {
- BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(pt)));
- try
- {
- retVal[0] = in.readLine();
- // skip all commented lines
- do {
- retVal[1] = in.readLine();
- } while ( retVal[1].charAt(0) == '%' );
-
- if ( !retVal[0].startsWith("%%") ) {
- raiseValidateError("MatrixMarket files must begin with a header line.", conditional);
- }
- }
- finally
- {
- if( in != null )
- in.close();
- }
- }
- else {
- raiseValidateError("Could not find the file: " + filename, conditional);
- }
-
- } catch (IOException e){
- //LOG.error(this.printErrorLocation() + "Error reading MatrixMarket file: " + filename );
- //throw new LanguageException(this.printErrorLocation() + "Error reading MatrixMarket file: " + filename );
- throw new LanguageException(e);
- }
-
- return retVal;
- }
-
- public boolean checkHasMatrixMarketFormat(String inputFileName, String mtdFileName, boolean conditional)
- throws LanguageException
- {
- // Check the MTD file exists. if there is an MTD file, return false.
- JSONObject mtdObject = readMetadataFile(mtdFileName, conditional);
-
- if (mtdObject != null)
- return false;
-
- boolean exists = false;
- FileSystem fs = null;
-
- try {
- fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- } catch (Exception e){
- LOG.error(this.printErrorLocation() + "could not read the configuration file.");
- throw new LanguageException(this.printErrorLocation() + "could not read the configuration file.", e);
- }
-
- Path pt = new Path(inputFileName);
- try {
- if (fs.exists(pt)){
- exists = true;
- }
- } catch (Exception e){
- LOG.error(this.printErrorLocation() + "file " + inputFileName + " not found");
- throw new LanguageException(this.printErrorLocation() + "file " + inputFileName + " not found");
- }
-
- try {
- // CASE: filename is a directory -- process as a directory
- if (exists && fs.getFileStatus(pt).isDirectory()){
-
- // currently, only MM files as files are supported. So, if file is directory, then infer
- // likely not MM file
- return false;
- }
- // CASE: filename points to a file
- else if (exists){
-
- //BufferedReader in = new BufferedReader(new FileReader(filename));
- BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(pt)));
-
- String headerLine = new String("");
- if (in.ready())
- headerLine = in.readLine();
- in.close();
-
- // check that headerline starts with "%%"
- // will infer malformed
- if( headerLine !=null && headerLine.startsWith("%%") )
- return true;
- else
- return false;
- }
- else {
- return false;
- }
-
- } catch (Exception e){
- return false;
- }
- }
-
- public boolean checkHasDelimitedFormat(String filename, boolean conditional)
- throws LanguageException
- {
-
- // if the MTD file exists, check the format is not binary
- JSONObject mtdObject = readMetadataFile(filename + ".mtd", conditional);
- if (mtdObject != null){
- String formatTypeString = (String)JSONHelper.get(mtdObject,FORMAT_TYPE);
- if (formatTypeString != null ) {
- if ( formatTypeString.equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV) )
- return true;
- else
- return false;
- }
- }
- return false;
-
- // The file format must be specified either in .mtd file or in read() statement
- // Therefore, one need not actually read the data to infer the format.
- }
-
- /**
- *
- * @return
- */
- public boolean isCSVReadWithUnknownSize()
- {
- boolean ret = false;
-
- E
<TRUNCATED>
[16/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
deleted file mode 100644
index 979cd22..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
+++ /dev/null
@@ -1,615 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-
-
-public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
-{
-
- private ParameterizedBuiltinFunctionOp _opcode;
- private HashMap<String,Expression> _varParams;
-
- public static final String TF_FN_PARAM_DATA = "target";
- public static final String TF_FN_PARAM_TXMTD = "transformPath";
- public static final String TF_FN_PARAM_TXSPEC = "transformSpec";
- public static final String TF_FN_PARAM_APPLYMTD = "applyTransformPath";
- public static final String TF_FN_PARAM_OUTNAMES = "outputNames";
-
- private static HashMap<String, Expression.ParameterizedBuiltinFunctionOp> opcodeMap;
- static {
- opcodeMap = new HashMap<String, Expression.ParameterizedBuiltinFunctionOp>();
- opcodeMap.put("aggregate", Expression.ParameterizedBuiltinFunctionOp.GROUPEDAGG);
- opcodeMap.put("groupedAggregate", Expression.ParameterizedBuiltinFunctionOp.GROUPEDAGG);
- opcodeMap.put("removeEmpty",Expression.ParameterizedBuiltinFunctionOp.RMEMPTY);
- opcodeMap.put("replace", Expression.ParameterizedBuiltinFunctionOp.REPLACE);
- opcodeMap.put("order", Expression.ParameterizedBuiltinFunctionOp.ORDER);
-
- // Distribution Functions
- opcodeMap.put("cdf", Expression.ParameterizedBuiltinFunctionOp.CDF);
- opcodeMap.put("pnorm", Expression.ParameterizedBuiltinFunctionOp.PNORM);
- opcodeMap.put("pt", Expression.ParameterizedBuiltinFunctionOp.PT);
- opcodeMap.put("pf", Expression.ParameterizedBuiltinFunctionOp.PF);
- opcodeMap.put("pchisq", Expression.ParameterizedBuiltinFunctionOp.PCHISQ);
- opcodeMap.put("pexp", Expression.ParameterizedBuiltinFunctionOp.PEXP);
-
- opcodeMap.put("icdf", Expression.ParameterizedBuiltinFunctionOp.INVCDF);
- opcodeMap.put("qnorm", Expression.ParameterizedBuiltinFunctionOp.QNORM);
- opcodeMap.put("qt", Expression.ParameterizedBuiltinFunctionOp.QT);
- opcodeMap.put("qf", Expression.ParameterizedBuiltinFunctionOp.QF);
- opcodeMap.put("qchisq", Expression.ParameterizedBuiltinFunctionOp.QCHISQ);
- opcodeMap.put("qexp", Expression.ParameterizedBuiltinFunctionOp.QEXP);
-
- // data transformation functions
- opcodeMap.put("transform", Expression.ParameterizedBuiltinFunctionOp.TRANSFORM);
- }
-
- public static HashMap<Expression.ParameterizedBuiltinFunctionOp, ParamBuiltinOp> pbHopMap;
- static {
- pbHopMap = new HashMap<Expression.ParameterizedBuiltinFunctionOp, ParamBuiltinOp>();
-
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.GROUPEDAGG, ParamBuiltinOp.GROUPEDAGG);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.RMEMPTY, ParamBuiltinOp.RMEMPTY);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.REPLACE, ParamBuiltinOp.REPLACE);
-
- // For order, a ReorgOp is constructed with ReorgOp.SORT type
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.ORDER, ParamBuiltinOp.INVALID);
-
- // Distribution Functions
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.CDF, ParamBuiltinOp.CDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.PNORM, ParamBuiltinOp.CDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.PT, ParamBuiltinOp.CDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.PF, ParamBuiltinOp.CDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.PCHISQ, ParamBuiltinOp.CDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.PEXP, ParamBuiltinOp.CDF);
-
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.INVCDF, ParamBuiltinOp.INVCDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.QNORM, ParamBuiltinOp.INVCDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.QT, ParamBuiltinOp.INVCDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.QF, ParamBuiltinOp.INVCDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.QCHISQ, ParamBuiltinOp.INVCDF);
- pbHopMap.put(Expression.ParameterizedBuiltinFunctionOp.QEXP, ParamBuiltinOp.INVCDF);
- }
-
- public static ParameterizedBuiltinFunctionExpression getParamBuiltinFunctionExpression(String functionName, ArrayList<ParameterExpression> paramExprsPassed,
- String fileName, int blp, int bcp, int elp, int ecp){
-
- if (functionName == null || paramExprsPassed == null)
- return null;
-
- Expression.ParameterizedBuiltinFunctionOp pbifop = opcodeMap.get(functionName);
-
- if ( pbifop == null )
- return null;
-
- HashMap<String,Expression> varParams = new HashMap<String,Expression>();
- for (ParameterExpression pexpr : paramExprsPassed)
- varParams.put(pexpr.getName(), pexpr.getExpr());
-
- ParameterizedBuiltinFunctionExpression retVal = new ParameterizedBuiltinFunctionExpression(pbifop,varParams,
- fileName, blp, bcp, elp, ecp);
- return retVal;
- } // end method getBuiltinFunctionExpression
-
-
- public ParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionOp op, HashMap<String,Expression> varParams,
- String filename, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.ParameterizedBuiltinFunctionOp;
- _opcode = op;
- _varParams = varParams;
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public ParameterizedBuiltinFunctionExpression(String filename, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.ParameterizedBuiltinFunctionOp;
- _opcode = ParameterizedBuiltinFunctionOp.INVALID;
- _varParams = new HashMap<String,Expression>();
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- HashMap<String,Expression> newVarParams = new HashMap<String,Expression>();
- for (String key : _varParams.keySet()){
- Expression newExpr = _varParams.get(key).rewriteExpression(prefix);
- newVarParams.put(key, newExpr);
- }
- ParameterizedBuiltinFunctionExpression retVal = new ParameterizedBuiltinFunctionExpression(_opcode, newVarParams,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- return retVal;
- }
-
- public void setOpcode(ParameterizedBuiltinFunctionOp op) {
- _opcode = op;
- }
-
- public ParameterizedBuiltinFunctionOp getOpCode() {
- return _opcode;
- }
-
- public HashMap<String,Expression> getVarParams() {
- return _varParams;
- }
-
- public Expression getVarParam(String name) {
- return _varParams.get(name);
- }
-
- public void addVarParam(String name, Expression value){
- _varParams.put(name, value);
- }
-
- public void removeVarParam(String name) {
- _varParams.remove(name);
- }
-
- /**
- * Validate parse tree : Process BuiltinFunction Expression in an assignment
- * statement
- *
- * @throws LanguageException
- */
- @Override
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- // validate all input parameters
- for ( String s : getVarParams().keySet() ) {
- Expression paramExpr = getVarParam(s);
-
- if (paramExpr instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
-
- paramExpr.validateExpression(ids, constVars, conditional);
- }
-
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- //output.setProperties(this.getFirstExpr().getOutput());
- this.setOutput(output);
-
- // IMPORTANT: for each operation, one must handle unnamed parameters
-
- switch (this.getOpCode()) {
-
- case GROUPEDAGG:
- validateGroupedAgg(output, conditional);
- break;
-
- case CDF:
- case INVCDF:
- case PNORM:
- case QNORM:
- case PT:
- case QT:
- case PF:
- case QF:
- case PCHISQ:
- case QCHISQ:
- case PEXP:
- case QEXP:
- validateDistributionFunctions(output, conditional);
- break;
-
- case RMEMPTY:
- validateRemoveEmpty(output, conditional);
- break;
-
- case REPLACE:
- validateReplace(output, conditional);
- break;
-
- case ORDER:
- validateOrder(output, conditional);
- break;
-
- case TRANSFORM:
- validateTransform(output, conditional);
- break;
-
- default: //always unconditional (because unsupported operation)
- raiseValidateError("Unsupported parameterized function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- return;
- }
-
- // example: A = transform(data=D, txmtd="", txspec="")
- private void validateTransform(DataIdentifier output, boolean conditional) throws LanguageException {
- Expression data = getVarParam(TF_FN_PARAM_DATA);
- if( data==null ) {
- raiseValidateError("Named parameter '" + TF_FN_PARAM_DATA + "' missing. Please specify the input data set.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( data.getOutput().getDataType() != DataType.FRAME ){
- raiseValidateError("Input to tansform() must be of type 'frame'. It is of type '"+data.getOutput().getDataType()+"'.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression txmtd = getVarParam(TF_FN_PARAM_TXMTD);
- if( txmtd==null ) {
- raiseValidateError("Named parameter '" + TF_FN_PARAM_TXMTD + "' missing. Please specify the transformation metadata file path.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( txmtd.getOutput().getDataType() != DataType.SCALAR || txmtd.getOutput().getValueType() != ValueType.STRING ){
- raiseValidateError("Transformation metadata file '" + TF_FN_PARAM_TXMTD + "' must be a string value (a scalar).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression txspec = getVarParam(TF_FN_PARAM_TXSPEC);
- Expression applyMTD = getVarParam(TF_FN_PARAM_APPLYMTD);
- if( txspec==null ) {
- if ( applyMTD == null )
- raiseValidateError("Named parameter '" + TF_FN_PARAM_TXSPEC + "' missing. Please specify the transformation specification file (in JSON format).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( txspec.getOutput().getDataType() != DataType.SCALAR || txspec.getOutput().getValueType() != ValueType.STRING ){
- raiseValidateError("Transformation specification file '" + TF_FN_PARAM_TXSPEC + "' must be a string value (a scalar).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- if ( applyMTD != null ) {
- if( applyMTD.getOutput().getDataType() != DataType.SCALAR || applyMTD.getOutput().getValueType() != ValueType.STRING ){
- raiseValidateError("Apply transformation metadata file'" + TF_FN_PARAM_APPLYMTD + "' must be a string value (a scalar).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- if(txspec != null ) {
- raiseValidateError("Only one of '" + TF_FN_PARAM_APPLYMTD + "' or '" + TF_FN_PARAM_TXSPEC + "' can be specified in transform().", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- Expression outNames = getVarParam(TF_FN_PARAM_OUTNAMES);
- if ( outNames != null ) {
- if( outNames.getOutput().getDataType() != DataType.SCALAR || outNames.getOutput().getValueType() != ValueType.STRING )
- raiseValidateError("The parameter specifying column names in the output file '" + TF_FN_PARAM_TXMTD + "' must be a string value (a scalar).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- if ( applyMTD != null)
- raiseValidateError("Only one of '" + TF_FN_PARAM_APPLYMTD + "' or '" + TF_FN_PARAM_OUTNAMES + "' can be specified in transform().", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // Output is a matrix with same dims as input
- output.setDataType(DataType.MATRIX);
- output.setFormatType(FormatType.CSV);
- output.setValueType(ValueType.DOUBLE);
- // Output dimensions may not be known at compile time, for example when dummycoding.
- output.setDimensions(-1, -1);
- }
-
- private void validateReplace(DataIdentifier output, boolean conditional) throws LanguageException {
- //check existence and correctness of arguments
- Expression target = getVarParam("target");
- if( target==null ) {
- raiseValidateError("Named parameter 'target' missing. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( target.getOutput().getDataType() != DataType.MATRIX ){
- raiseValidateError("Input matrix 'target' is of type '"+target.getOutput().getDataType()+"'. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression pattern = getVarParam("pattern");
- if( pattern==null ) {
- raiseValidateError("Named parameter 'pattern' missing. Please specify the replacement pattern.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( pattern.getOutput().getDataType() != DataType.SCALAR ){
- raiseValidateError("Replacement pattern 'pattern' is of type '"+pattern.getOutput().getDataType()+"'. Please, specify a scalar replacement pattern.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression replacement = getVarParam("replacement");
- if( replacement==null ) {
- raiseValidateError("Named parameter 'replacement' missing. Please specify the replacement value.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( replacement.getOutput().getDataType() != DataType.SCALAR ){
- raiseValidateError("Replacement value 'replacement' is of type '"+replacement.getOutput().getDataType()+"'. Please, specify a scalar replacement value.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // Output is a matrix with same dims as input
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(target.getOutput().getDim1(), target.getOutput().getDim2());
- }
-
- private void validateOrder(DataIdentifier output, boolean conditional) throws LanguageException {
- //check existence and correctness of arguments
- Expression target = getVarParam("target"); //[MANDATORY] TARGET
- if( target==null ) {
- raiseValidateError("Named parameter 'target' missing. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( target.getOutput().getDataType() != DataType.MATRIX ){
- raiseValidateError("Input matrix 'target' is of type '"+target.getOutput().getDataType()+"'. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- //check for unsupported parameters
- for(String param : getVarParams().keySet())
- if( !(param.equals("target") || param.equals("by") || param.equals("decreasing") || param.equals("index.return")) )
- raiseValidateError("Unsupported order parameter: '"+param+"'", false);
-
- Expression orderby = getVarParam("by"); //[OPTIONAL] BY
- if( orderby == null ) { //default first column, good fit for vectors
- orderby = new IntIdentifier(1, "1", -1, -1, -1, -1);
- addVarParam("by", orderby);
- }
- else if( orderby !=null && orderby.getOutput().getDataType() != DataType.SCALAR ){
- raiseValidateError("Orderby column 'by' is of type '"+orderby.getOutput().getDataType()+"'. Please, specify a scalar order by column index.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression decreasing = getVarParam("decreasing"); //[OPTIONAL] DECREASING
- if( decreasing == null ) { //default: ascending
- addVarParam("decreasing", new BooleanIdentifier(false, "false", -1, -1, -1, -1));
- }
- else if( decreasing!=null && decreasing.getOutput().getDataType() != DataType.SCALAR ){
- raiseValidateError("Ordering 'decreasing' is of type '"+decreasing.getOutput().getDataType()+"', '"+decreasing.getOutput().getValueType()+"'. Please, specify 'decreasing' as a scalar boolean.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression indexreturn = getVarParam("index.return"); //[OPTIONAL] DECREASING
- if( indexreturn == null ) { //default: sorted data
- indexreturn = new BooleanIdentifier(false, "false", -1, -1, -1, -1);
- addVarParam("index.return", indexreturn);
- }
- else if( indexreturn!=null && indexreturn.getOutput().getDataType() != DataType.SCALAR ){
- raiseValidateError("Return type 'index.return' is of type '"+indexreturn.getOutput().getDataType()+"', '"+indexreturn.getOutput().getValueType()+"'. Please, specify 'indexreturn' as a scalar boolean.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- long dim2 = ( indexreturn instanceof BooleanIdentifier ) ?
- ((BooleanIdentifier)indexreturn).getValue() ? 1: target.getOutput().getDim2() : -1;
-
- // Output is a matrix with same dims as input
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(target.getOutput().getDim1(), dim2 );
-
- }
-
- private void validateRemoveEmpty(DataIdentifier output, boolean conditional) throws LanguageException {
- //check existence and correctness of arguments
- Expression target = getVarParam("target");
- if( target==null ) {
- raiseValidateError("Named parameter 'target' missing. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( target.getOutput().getDataType() != DataType.MATRIX ){
- raiseValidateError("Input matrix 'target' is of type '"+target.getOutput().getDataType()+"'. Please specify the input matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression margin = getVarParam("margin");
- if( margin==null ){
- raiseValidateError("Named parameter 'margin' missing. Please specify 'rows' or 'cols'.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if( !(margin instanceof DataIdentifier) && !margin.toString().equals("rows") && !margin.toString().equals("cols") ){
- raiseValidateError("Named parameter 'margin' has an invalid value '"+margin.toString()+"'. Please specify 'rows' or 'cols'.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- Expression select = getVarParam("select");
- if( select!=null && select.getOutput().getDataType() != DataType.MATRIX ){
- raiseValidateError("Index matrix 'select' is of type '"+select.getOutput().getDataType()+"'. Please specify the select matrix.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // Output is a matrix with unknown dims
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(-1, -1);
-
- }
-
- private void validateGroupedAgg(DataIdentifier output, boolean conditional) throws LanguageException {
- int colwise = -1;
- if (getVarParam(Statement.GAGG_TARGET) == null || getVarParam(Statement.GAGG_GROUPS) == null){
- raiseValidateError("Must define both target and groups and both must have same dimensions", conditional);
- }
- if (getVarParam(Statement.GAGG_TARGET) instanceof DataIdentifier && getVarParam(Statement.GAGG_GROUPS) instanceof DataIdentifier && (getVarParam(Statement.GAGG_WEIGHTS) == null || getVarParam(Statement.GAGG_WEIGHTS) instanceof DataIdentifier))
- {
-
- DataIdentifier targetid = (DataIdentifier)getVarParam(Statement.GAGG_TARGET);
- DataIdentifier groupsid = (DataIdentifier)getVarParam(Statement.GAGG_GROUPS);
- DataIdentifier weightsid = (DataIdentifier)getVarParam(Statement.GAGG_WEIGHTS);
-
- if ( targetid.dimsKnown() ) {
- colwise = targetid.getDim1() > targetid.getDim2() ? 1 : 0;
- }
- else if ( groupsid.dimsKnown() ) {
- colwise = groupsid.getDim1() > groupsid.getDim2() ? 1 : 0;
- }
- else if ( weightsid != null && weightsid.dimsKnown() ) {
- colwise = weightsid.getDim1() > weightsid.getDim2() ? 1 : 0;
- }
-
- //precompute number of rows and columns because target can be row or column vector
- long rowsTarget = targetid.getDim1(); // Math.max(targetid.getDim1(),targetid.getDim2());
- long colsTarget = targetid.getDim2(); // Math.min(targetid.getDim1(),targetid.getDim2());
-
- if( targetid.dimsKnown() && groupsid.dimsKnown() &&
- (rowsTarget != groupsid.getDim1() || colsTarget != groupsid.getDim2()) )
- {
- raiseValidateError("target and groups must have same dimensions -- "
- + " targetid dims: " + targetid.getDim1() +" rows, " + targetid.getDim2() + " cols -- groupsid dims: " + groupsid.getDim1() + " rows, " + groupsid.getDim2() + " cols ", conditional);
- }
-
- if( weightsid != null && (targetid.dimsKnown() && weightsid.dimsKnown()) &&
- (rowsTarget != weightsid.getDim1() || colsTarget != weightsid.getDim2() ))
- {
- raiseValidateError("target and weights must have same dimensions -- "
- + " targetid dims: " + targetid.getDim1() +" rows, " + targetid.getDim2() + " cols -- weightsid dims: " + weightsid.getDim1() + " rows, " + weightsid.getDim2() + " cols ", conditional);
- }
- }
-
-
- if (getVarParam(Statement.GAGG_FN) == null){
- raiseValidateError("must define function name (fn=<function name>) for aggregate()", conditional);
- }
-
- Expression functParam = getVarParam(Statement.GAGG_FN);
-
- if (functParam instanceof Identifier)
- {
- // standardize to lowercase and dequote fname
- String fnameStr = getVarParam(Statement.GAGG_FN).toString();
-
-
- // check that IF fname="centralmoment" THEN order=m is defined, where m=2,3,4
- // check ELSE IF fname is allowed
- if(fnameStr.equals(Statement.GAGG_FN_CM)){
- String orderStr = getVarParam(Statement.GAGG_FN_CM_ORDER) == null ? null : getVarParam(Statement.GAGG_FN_CM_ORDER).toString();
- if (orderStr == null || !(orderStr.equals("2") || orderStr.equals("3") || orderStr.equals("4"))){
- raiseValidateError("for centralmoment, must define order. Order must be equal to 2,3, or 4", conditional);
- }
- }
- else if (fnameStr.equals(Statement.GAGG_FN_COUNT)
- || fnameStr.equals(Statement.GAGG_FN_SUM)
- || fnameStr.equals(Statement.GAGG_FN_MEAN)
- || fnameStr.equals(Statement.GAGG_FN_VARIANCE)){}
- else {
- raiseValidateError("fname is " + fnameStr + " but must be either centeralmoment, count, sum, mean, variance", conditional);
- }
- }
-
- Expression ngroupsParam = getVarParam(Statement.GAGG_NUM_GROUPS);
- long outputDim1 = -1, outputDim2 = -1;
- if( ngroupsParam != null && ngroupsParam instanceof Identifier )
- {
- Identifier numGroups = (Identifier) ngroupsParam;
- if ( numGroups != null && numGroups instanceof ConstIdentifier) {
- long ngroups = ((ConstIdentifier)numGroups).getLongValue();
- if ( colwise == 1 ) {
- outputDim1 = ngroups;
- outputDim2 = 1;
- }
- else if ( colwise == 0 ) {
- outputDim1 = 1;
- outputDim2 = ngroups;
- }
- }
- }
-
- // Output is a matrix with unknown dims
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(outputDim1, outputDim2);
- }
-
- private void validateDistributionFunctions(DataIdentifier output, boolean conditional) throws LanguageException {
- // CDF and INVCDF expects one unnamed parameter, it must be renamed as "quantile"
- // (i.e., we must compute P(X <= x) where x is called as "quantile" )
-
- ParameterizedBuiltinFunctionOp op = this.getOpCode();
-
- // check if quantile is of type SCALAR
- if ( getVarParam("target") == null || getVarParam("target").getOutput().getDataType() != DataType.SCALAR ) {
- raiseValidateError("target must be provided for distribution functions, and it must be a scalar value.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- // Distribution specific checks
- switch(op) {
- case CDF:
- case INVCDF:
- if(getVarParam("dist") == null) {
- raiseValidateError("For cdf() and icdf(), a distribution function must be specified (as a string).", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- break;
-
- case QF:
- case PF:
- if(getVarParam("df1") == null || getVarParam("df2") == null ) {
- raiseValidateError("Two degrees of freedom df1 and df2 must be provided for F-distribution.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- break;
-
- case QT:
- case PT:
- if(getVarParam("df") == null ) {
- raiseValidateError("Degrees of freedom df must be provided for t-distribution.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- break;
-
- case QCHISQ:
- case PCHISQ:
- if(getVarParam("df") == null ) {
- raiseValidateError("Degrees of freedom df must be provided for chi-squared-distribution.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- break;
-
- default:
- break;
-
- // Not checking for QNORM, PNORM: distribution parameters mean and sd are optional with default values 0.0 and 1.0, respectively
- // Not checking for QEXP, PEXP: distribution parameter rate is optional with a default values 1.0
-
- // For all cdf functions, additional parameter lower.tail is optional with a default value TRUE
- }
-
- // CDF and INVCDF specific checks:
- switch(op) {
- case INVCDF:
- case QNORM:
- case QF:
- case QT:
- case QCHISQ:
- case QEXP:
- if(getVarParam("lower.tail") != null ) {
- raiseValidateError("Lower tail argument is invalid while computing inverse cumulative probabilities.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- break;
-
- case CDF:
- case PNORM:
- case PF:
- case PT:
- case PCHISQ:
- case PEXP:
- // no checks yet
- break;
-
- default:
- break;
- }
-
- // Output is a scalar
- output.setDataType(DataType.SCALAR);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(0, 0);
- return;
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder(_opcode.toString() + "(");
-
- for (String key : _varParams.keySet()){
- sb.append("," + key + "=" + _varParams.get(key));
- }
- sb.append(" )");
- return sb.toString();
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- for (String s : _varParams.keySet()) {
- result.addVariables ( _varParams.get(s).variablesRead() );
- }
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- for (String s : _varParams.keySet()) {
- result.addVariables ( _varParams.get(s).variablesUpdated() );
- }
- result.addVariable(((DataIdentifier)this.getOutput()).getName(), (DataIdentifier)this.getOutput());
- return result;
- }
-
- @Override
- public boolean multipleReturns() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParseException.java b/src/main/java/com/ibm/bi/dml/parser/ParseException.java
deleted file mode 100644
index 86ff297..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ParseException.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-/**
- * This exception is thrown when parse errors are encountered.
- * You can explicitly create objects of this exception type by
- * calling the method generateParseException in the generated
- * parser.
- *
- * You can modify this class to customize your error reporting
- * mechanisms so long as you retain the public fields.
- *
- *
- * NOTE: MB: Originally, this exception was generated with javacc.
- * Unfortunately, many classes directly use this exception. Hence,
- * on removing javacc we kept a condensed version of it.
- * TODO: eventually we should remove the token class and create our
- * own exception or just use LanguageException.
- */
-public class ParseException extends Exception
-{
-
- /**
- * The version identifier for this Serializable class.
- * Increment only if the <i>serialized</i> form of the
- * class changes.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The following constructors are for use by you for whatever
- * purpose you can think of. Constructing the exception in this
- * manner makes the exception behave in the normal way - i.e., as
- * documented in the class "Throwable". The fields "errorToken",
- * "expectedTokenSequences", and "tokenImage" do not contain
- * relevant information. The JavaCC generated code does not use
- * these constructors.
- */
-
- public ParseException() {
- super();
- }
-
- /** Constructor with message. */
- public ParseException(String message) {
- super(message);
- }
-
- /**
- * This is the last token that has been consumed successfully. If
- * this object has been created due to a parse error, the token
- * following this token will (therefore) be the first error token.
- */
- public Token currentToken;
-
- /**
- * Each entry in this array is an array of integers. Each array
- * of integers represents a sequence of tokens (by their ordinal
- * values) that is expected at this point of the parse.
- */
- public int[][] expectedTokenSequences;
-
- /**
- * This is a reference to the "tokenImage" array of the generated
- * parser within which the parse error occurred. This array is
- * defined in the generated ...Constants interface.
- */
- public String[] tokenImage;
-
-}
-/* JavaCC - OriginalChecksum=f4855557e750c030aaf6a53d0d8438bb (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/PathStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/PathStatement.java b/src/main/java/com/ibm/bi/dml/parser/PathStatement.java
deleted file mode 100644
index 9a9280e..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/PathStatement.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class PathStatement extends Statement
-{
-
- // Set of file system paths to packages
- private String _pathValue;
-
- public PathStatement(String pv){
- _pathValue = pv;
- }
-
- public String getPathValue(){
- return _pathValue;
- }
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- return this;
- }
-
- public void initializeforwardLV(VariableSet activeIn){}
-
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public boolean controlStatement() {
- return false;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(Statement.SETWD + "(");
- sb.append(_pathValue);
- sb.append(")");
- sb.append(";");
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/PrintStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/PrintStatement.java b/src/main/java/com/ibm/bi/dml/parser/PrintStatement.java
deleted file mode 100644
index aac58ea..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/PrintStatement.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-
-
-public class PrintStatement extends Statement
-{
- public enum PRINTTYPE {PRINT, STOP};
-
- protected PRINTTYPE _type; // print or stop
- protected Expression _expr;
-
- private static PRINTTYPE getPrintType(String type) throws LanguageException {
- if(type.equalsIgnoreCase("print")) {
- return PRINTTYPE.PRINT;
- }
- else if (type.equalsIgnoreCase("stop")) {
- return PRINTTYPE.STOP;
- }
- else
- throw new LanguageException("Unknown statement type: " + type);
- }
-
- public PrintStatement(String type, Expression expr, int beginLine, int beginCol, int endLine, int endCol)
- throws LanguageException
- {
- this(getPrintType(type), expr);
-
- setBeginLine(beginLine);
- setBeginColumn(beginCol);
- setEndLine(endLine);
- setEndColumn(endCol);
- }
-
- public PrintStatement(PRINTTYPE type, Expression expr) throws LanguageException{
- _type = type;
- _expr = expr;
- }
-
- public Statement rewriteStatement(String prefix) throws LanguageException{
- Expression newExpr = _expr.rewriteExpression(prefix);
- PrintStatement retVal = new PrintStatement(_type, newExpr);
- retVal.setBeginLine(this.getBeginLine());
- retVal.setBeginColumn(this.getBeginColumn());
- retVal.setEndLine(this.getEndLine());
- retVal.setEndColumn(this.getEndColumn());
-
- return retVal;
- }
-
- public void initializeforwardLV(VariableSet activeIn){}
-
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- sb.append(_type + " (" );
- if (_expr != null){
- sb.append(_expr.toString());
- }
- sb.append(");");
- return sb.toString();
- }
-
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = _expr.variablesRead();
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return new VariableSet();
- }
-
- @Override
- public boolean controlStatement() {
- // ensure that breakpoints end up in own statement block
- if (DMLScript.ENABLE_DEBUG_MODE) {
- DMLBreakpointManager.insertBreakpoint(_expr.getBeginLine());
- return true;
- }
-
- // Keep stop() statement in a separate statement block
- if(getType() == PRINTTYPE.STOP)
- return true;
-
- return false;
- }
-
- public Expression getExpression(){
- return _expr;
- }
-
- public PRINTTYPE getType() {
- return _type;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/RelationalExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/RelationalExpression.java b/src/main/java/com/ibm/bi/dml/parser/RelationalExpression.java
deleted file mode 100644
index 06f76f2..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/RelationalExpression.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-
-public class RelationalExpression extends Expression
-{
-
- private Expression _left;
- private Expression _right;
- private RelationalOp _opcode;
-
- public RelationalExpression(RelationalOp bop) {
- _kind = Kind.RelationalOp;
- _opcode = bop;
-
- setFilename("MAIN SCRIPT");
- setBeginLine(0);
- setBeginColumn(0);
- setEndLine(0);
- setEndColumn(0);
- }
-
- public RelationalExpression(RelationalOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn) {
- _kind = Kind.RelationalOp;
- _opcode = bop;
-
- setFilename(filename);
- setBeginLine(beginLine);
- setBeginColumn(beginColumn);
- setEndLine(endLine);
- setEndColumn(endColumn);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
-
- RelationalExpression newExpr = new RelationalExpression(this._opcode, getFilename(), getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn());
- newExpr.setLeft(_left.rewriteExpression(prefix));
- newExpr.setRight(_right.rewriteExpression(prefix));
- return newExpr;
- }
-
- public RelationalOp getOpCode(){
- return _opcode;
- }
-
- public void setLeft(Expression l){
- _left = l;
-
- // update script location information --> left expression is BEFORE in script
- if (_left != null){
- setFilename(_left.getFilename());
- setBeginLine(_left.getBeginLine());
- setBeginColumn(_left.getBeginColumn());
- }
-
- }
-
- public void setRight(Expression r){
- _right = r;
-
- // update script location information --> right expression is AFTER in script
- if (_right != null){
- setFilename(_right.getFilename());
- setBeginLine(_right.getEndLine());
- setBeginColumn(_right.getEndColumn());
- }
- }
-
- public Expression getLeft(){
- return _left;
- }
-
- public Expression getRight(){
- return _right;
- }
-
- /**
- * Validate parse tree : Process Relational Expression
- * @throws LanguageException
- */
- @Override
- public void validateExpression(HashMap<String,DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- //check for functions calls in expression
- if (_left instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in relational expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
- if (_right instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in relational expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- // handle <NUMERIC> == <BOOLEAN> --> convert <BOOLEAN> to numeric value
- if ((_left != null && _left instanceof BooleanIdentifier) || (_right != null && _right instanceof BooleanIdentifier)){
- if ((_left instanceof IntIdentifier || _left instanceof DoubleIdentifier) || _right instanceof IntIdentifier || _right instanceof DoubleIdentifier){
- if (_left instanceof BooleanIdentifier){
- if (((BooleanIdentifier) _left).getValue())
- this.setLeft(new IntIdentifier(1, _left.getFilename(), _left.getBeginLine(), _left.getBeginColumn(), _left.getEndLine(), _left.getEndColumn()));
- else
- this.setLeft(new IntIdentifier(0, _left.getFilename(), _left.getBeginLine(), _left.getBeginColumn(), _left.getEndLine(), _left.getEndColumn()));
- }
- else if (_right instanceof BooleanIdentifier){
- if (((BooleanIdentifier) _right).getValue())
- this.setRight(new IntIdentifier(1, _right.getFilename(), _right.getBeginLine(), _right.getBeginColumn(), _right.getEndLine(),_right.getEndColumn()));
- else
- this.setRight(new IntIdentifier(0, _right.getFilename(), _right.getBeginLine(), _right.getBeginColumn(), _right.getEndLine(),_right.getEndColumn()));
- }
- }
- }
-
- //recursive validate
- _left.validateExpression(ids, constVars, conditional);
- if( _right !=null )
- _right.validateExpression(ids, constVars, conditional);
-
- //constant propagation (precondition for more complex constant folding rewrite)
- if( _left instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _left).getName()) )
- _left = constVars.get(((DataIdentifier) _left).getName());
- if( _right instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _right).getName()) )
- _right = constVars.get(((DataIdentifier) _right).getName());
-
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- boolean isLeftMatrix = (_left.getOutput() != null && _left.getOutput().getDataType() == DataType.MATRIX);
- boolean isRightMatrix = (_right.getOutput() != null && _right.getOutput().getDataType() == DataType.MATRIX);
- if(isLeftMatrix || isRightMatrix) {
- // Added to support matrix relational comparison
- if(isLeftMatrix && isRightMatrix) {
- checkMatchingDimensions(_left, _right, true);
- }
-
- long[] dims = getBinaryMatrixCharacteristics(_left, _right);
- output.setDataType(DataType.MATRIX);
- output.setDimensions(dims[0], dims[1]);
- output.setBlockDimensions(dims[2], dims[3]);
-
- //since SystemML only supports double matrices, the value type is forced to
- //double; once we support boolean matrices this needs to change
- output.setValueType(ValueType.DOUBLE);
- }
- else {
- output.setBooleanProperties();
- }
-
- this.setOutput(output);
- }
-
- /**
- * This is same as the function from BuiltinFunctionExpression which is called by ppred
- * @param expr1
- * @param expr2
- * @throws LanguageException
- */
- private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV)
- throws LanguageException
- {
- if (expr1 != null && expr2 != null) {
-
- // if any matrix has unknown dimensions, simply return
- if( expr1.getOutput().getDim1() == -1 || expr2.getOutput().getDim1() == -1
- ||expr1.getOutput().getDim2() == -1 || expr2.getOutput().getDim2() == -1 )
- {
- return;
- }
- else if( (!allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1())
- || (allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1() && expr2.getOutput().getDim1() != 1)
- || (!allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2())
- || (allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2() && expr2.getOutput().getDim2() != 1) )
- {
- raiseValidateError("Mismatch in matrix dimensions of parameters for function "
- + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
-
- public String toString(){
- return "(" + _left.toString() + " " + _opcode.toString() + " " + _right.toString() + ")";
- }
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesRead());
- result.addVariables(_right.variablesRead());
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesUpdated());
- result.addVariables(_right.variablesUpdated());
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/Statement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/Statement.java b/src/main/java/com/ibm/bi/dml/parser/Statement.java
deleted file mode 100644
index bf76ce0..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/Statement.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-public abstract class Statement
-{
-
-
- protected static final Log LOG = LogFactory.getLog(Statement.class.getName());
-
- public static final String OUTPUTSTATEMENT = "write";
-
- // parameter names for seq()
- public static final String SEQ_FROM = "from";
- public static final String SEQ_TO = "to";
- public static final String SEQ_INCR = "incr";
-
- public static final String SOURCE = "source";
- public static final String SETWD = "setwd";
-
- public static final String MATRIX_DATA_TYPE = "matrix";
- public static final String FRAME_DATA_TYPE = "frame";
- public static final String SCALAR_DATA_TYPE = "scalar";
-
- public static final String DOUBLE_VALUE_TYPE = "double";
- public static final String BOOLEAN_VALUE_TYPE = "boolean";
- public static final String INT_VALUE_TYPE = "int";
- public static final String STRING_VALUE_TYPE = "string";
-
- // String constants related to Grouped Aggregate parameters
- public static final String GAGG_TARGET = "target";
- public static final String GAGG_GROUPS = "groups";
- public static final String GAGG_WEIGHTS = "weights";
- public static final String GAGG_FN = "fn";
- public static final String GAGG_FN_SUM = "sum";
- public static final String GAGG_FN_COUNT = "count";
- public static final String GAGG_FN_MEAN = "mean";
- public static final String GAGG_FN_VARIANCE = "variance";
- public static final String GAGG_FN_CM = "centralmoment";
- public static final String GAGG_FN_CM_ORDER = "order";
- public static final String GAGG_NUM_GROUPS = "ngroups";
-
- public abstract boolean controlStatement();
-
- public abstract VariableSet variablesRead();
- public abstract VariableSet variablesUpdated();
-
- public abstract void initializeforwardLV(VariableSet activeIn) throws LanguageException;
- public abstract VariableSet initializebackwardLV(VariableSet lo) throws LanguageException;
-
- public abstract Statement rewriteStatement(String prefix) throws LanguageException;
-
- // Used only insider python parser to allow for ignoring newline logic
- private boolean isEmptyNewLineStatement = false;
- public boolean isEmptyNewLineStatement() {
- return isEmptyNewLineStatement;
- }
- public void setEmptyNewLineStatement(boolean isEmptyNewLineStatement) {
- this.isEmptyNewLineStatement = isEmptyNewLineStatement;
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store exception info + position information for statements
- ///////////////////////////////////////////////////////////////////////////
-
-
- private String _filename;
- private int _beginLine, _beginColumn;
- private int _endLine, _endColumn;
-
- public void setFilename(String passed) { _filename = passed; }
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed;}
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(String filename, int blp, int bcp, int elp, int ecp){
- _filename = filename;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
- public String getFilename() { return _filename; }
-
- public String printErrorLocation(){
- return "ERROR: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printInfoLocation(){
- return "INFO: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printErrorLocation(int beginLine, int beginColumn){
- return "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public String printWarningLocation(int beginLine, int beginColumn){
- return "WARNING: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public String printInfoLocation(int beginLine, int beginColumn){
- return "INFO: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
-
-}
[10/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ParForProgramBlock.java
deleted file mode 100644
index faf85c9..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ParForProgramBlock.java
+++ /dev/null
@@ -1,2126 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Level;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerLocal;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerRemoteMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.LocalParWorker;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.LocalTaskQueue;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ParForBody;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteDPParForMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteDPParForSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForJobReturn;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.RemoteParForSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMerge;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalAutomatic;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalFile;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeLocalMemory;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeRemoteMR;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ResultMergeRemoteSpark;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitioner;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoring;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmax;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFactoringCmin;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerFixedsize;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerNaive;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.TaskPartitionerStatic;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimator;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTree;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTreeConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizationWrapper;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.ProgramRecompiler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Stat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
-
-
-
-/**
- * The ParForProgramBlock has the same execution semantics as a ForProgamBlock but executes
- * the independent iterations in parallel. See ParForStatementBlock for the loop dependency
- * analysis. At runtime level, iterations are guaranteed to be completely independent.
- *
- * NEW FUNCTIONALITIES (not for BI 2.0 release)
- * TODO: reduction variables (operations: +=, -=, /=, *=, min, max)
- * TODO: papply(A,1:2,FUN) language construct (compiled to ParFOR) via DML function repository => modules OK, but second-order functions required
- *
- */
-public class ParForProgramBlock extends ForProgramBlock
-{
- // execution modes
- public enum PExecMode {
- LOCAL, //local (master) multi-core execution mode
- REMOTE_MR, //remote (MR cluster) execution mode
- REMOTE_MR_DP, //remote (MR cluster) execution mode, fused with data partitioning
- REMOTE_SPARK, //remote (Spark cluster) execution mode
- REMOTE_SPARK_DP,//remote (Spark cluster) execution mode, fused with data partitioning
- UNSPECIFIED
- }
-
- // task partitioner
- public enum PTaskPartitioner {
- FIXED, //fixed-sized task partitioner, uses tasksize
- NAIVE, //naive task partitioner (tasksize=1)
- STATIC, //static task partitioner (numIterations/numThreads)
- FACTORING, //factoring task partitioner
- FACTORING_CMIN, //constrained factoring task partitioner, uses tasksize as min constraint
- FACTORING_CMAX, //constrained factoring task partitioner, uses tasksize as max constraint
- UNSPECIFIED
- }
-
- public enum PDataPartitionFormat {
- NONE,
- ROW_WISE,
- ROW_BLOCK_WISE,
- ROW_BLOCK_WISE_N,
- COLUMN_WISE,
- COLUMN_BLOCK_WISE,
- COLUMN_BLOCK_WISE_N,
- BLOCK_WISE_M_N,
- UNSPECIFIED;
-
- /**
- * Note: Robust version of valueOf in order to return NONE without exception
- * if misspelled or non-existing and for case-insensitivity.
- *
- * @param s
- * @return
- */
- public static PDataPartitionFormat parsePDataPartitionFormat(String s) {
- if (s.equalsIgnoreCase("ROW_WISE"))
- return ROW_WISE;
- else if (s.equalsIgnoreCase("ROW_BLOCK_WISE"))
- return ROW_BLOCK_WISE;
- else if (s.equalsIgnoreCase("ROW_BLOCK_WISE_N"))
- return ROW_BLOCK_WISE_N;
- else if (s.equalsIgnoreCase("COLUMN_WISE"))
- return COLUMN_WISE;
- else if (s.equalsIgnoreCase("COLUMN_BLOCK_WISE"))
- return COLUMN_BLOCK_WISE;
- else if (s.equalsIgnoreCase("COLUMN_BLOCK_WISE_N"))
- return COLUMN_BLOCK_WISE_N;
- else if (s.equalsIgnoreCase("BLOCK_WISE_M_N"))
- return BLOCK_WISE_M_N;
- else
- return NONE;
- }
- }
-
- public enum PDataPartitioner {
- NONE, // no data partitioning
- LOCAL, // local file based partition split on master node
- REMOTE_MR, // remote partition split using a reblock MR job
- REMOTE_SPARK, // remote partition split using a spark job
- UNSPECIFIED,
- }
-
- public enum PResultMerge {
- LOCAL_MEM, // in-core (in-memory) result merge (output and one input at a time)
- LOCAL_FILE, // out-of-core result merge (file format dependent)
- LOCAL_AUTOMATIC, // decides between MEM and FILE based on the size of the output matrix
- REMOTE_MR, // remote MR parallel result merge
- REMOTE_SPARK, // remote Spark parallel result merge
- UNSPECIFIED,
- }
-
- //optimizer
- public enum POptMode{
- NONE, //no optimization, use defaults and specified parameters
- RULEBASED, //some simple rule-based rewritings (affects only parfor PB) - similar to HEURISTIC but no exec time estimates
- CONSTRAINED, //same as rule-based but with given params as constraints
- HEURISTIC, //some simple cost-based rewritings (affects only parfor PB)
- GREEDY, //greedy cost-based optimization algorithm (potentially local optimum, affects all instructions)
- FULL_DP, //full cost-based optimization algorithm (global optimum, affects all instructions)
- }
-
- // internal parameters
- public static final boolean OPTIMIZE = true; // run all automatic optimizations on top-level parfor
- public static final boolean USE_PB_CACHE = false; // reuse copied program blocks whenever possible, not there can be issues related to recompile
- public static boolean USE_RANGE_TASKS_IF_USEFUL = true; // use range tasks whenever size>3, false, otherwise wrong split order in remote
- public static final boolean USE_STREAMING_TASK_CREATION = true; // start working while still creating tasks, prevents blocking due to too small task queue
- public static final boolean ALLOW_NESTED_PARALLELISM = true; // if not, transparently change parfor to for on program conversions (local,remote)
- public static boolean ALLOW_REUSE_MR_JVMS = true; // potential benefits: less setup costs per task, NOTE> cannot be used MR4490 in Hadoop 1.0.3, still not fixed in 1.1.1
- public static boolean ALLOW_REUSE_MR_PAR_WORKER = ALLOW_REUSE_MR_JVMS; //potential benefits: less initialization, reuse in-memory objects and result consolidation!
- public static final boolean USE_FLEX_SCHEDULER_CONF = false;
- public static final boolean USE_PARALLEL_RESULT_MERGE = false; // if result merge is run in parallel or serial
- public static final boolean USE_PARALLEL_RESULT_MERGE_REMOTE = true; // if remote result merge should be run in parallel for multiple result vars
- public static final boolean ALLOW_DATA_COLOCATION = true;
- public static final boolean CREATE_UNSCOPED_RESULTVARS = true;
- public static boolean ALLOW_REUSE_PARTITION_VARS = true; //reuse partition input matrices, applied only if read-only in surrounding loops
- public static final int WRITE_REPLICATION_FACTOR = 1;
- public static final int MAX_RETRYS_ON_ERROR = 1;
- public static final boolean FORCE_CP_ON_REMOTE_MR = true; // compile body to CP if exec type forced to MR
- public static final boolean LIVEVAR_AWARE_EXPORT = true; //export only read variables according to live variable analysis
- public static final boolean LIVEVAR_AWARE_CLEANUP = true; //cleanup pinned variables according to live variable analysis
- public static final boolean RESET_RECOMPILATION_FLAGs = true;
-
- public static final String PARFOR_FNAME_PREFIX = "/parfor/";
- public static final String PARFOR_MR_TASKS_TMP_FNAME = PARFOR_FNAME_PREFIX + "%ID%_MR_taskfile";
- public static final String PARFOR_MR_RESULT_TMP_FNAME = PARFOR_FNAME_PREFIX + "%ID%_MR_results";
- public static final String PARFOR_MR_RESULTMERGE_FNAME = PARFOR_FNAME_PREFIX + "%ID%_resultmerge%VAR%";
- public static final String PARFOR_DATAPARTITIONS_FNAME = PARFOR_FNAME_PREFIX + "%ID%_datapartitions%VAR%";
-
- public static final String PARFOR_COUNTER_GROUP_NAME = "SystemML ParFOR Counters";
-
- // static ID generator sequences
- private static IDSequence _pfIDSeq = null;
- private static IDSequence _pwIDSeq = null;
-
- // runtime parameters
- protected HashMap<String,String> _params = null;
- protected int _numThreads = -1;
- protected PTaskPartitioner _taskPartitioner = null;
- protected long _taskSize = -1;
- protected PDataPartitioner _dataPartitioner = null;
- protected PResultMerge _resultMerge = null;
- protected PExecMode _execMode = null;
- protected POptMode _optMode = null;
- protected boolean _monitor = false;
- protected Level _optLogLevel = null;
-
-
- //specifics used for optimization
- protected long _numIterations = -1;
- protected String[] _iterablePredicateVarsOriginal = null;
-
- //specifics used for data partitioning
- protected LocalVariableMap _variablesDPOriginal = null;
- protected LocalVariableMap _variablesDPReuse = null;
- protected String _colocatedDPMatrix = null;
- protected boolean _tSparseCol = false;
- protected int _replicationDP = WRITE_REPLICATION_FACTOR;
- protected int _replicationExport = -1;
- //specifics used for result partitioning
- protected boolean _jvmReuse = true;
- //specifics used for recompilation
- protected double _oldMemoryBudget = -1;
- protected double _recompileMemoryBudget = -1;
- //specifics for caching
- protected boolean _enableCPCaching = true;
- protected boolean _enableRuntimePiggybacking = false;
- //specifics for spark
- protected Collection<String> _variablesRP = null;
- protected Collection<String> _variablesECache = null;
-
- // program block meta data
- protected long _ID = -1;
- protected int _IDPrefix = -1;
- protected ArrayList<String> _resultVars = null;
- protected IDSequence _resultVarsIDSeq = null;
- protected IDSequence _dpVarsIDSeq = null;
- protected boolean _monitorReport = false;
- protected boolean _hasFunctions = true;
-
- // local parworker data
- protected long[] _pwIDs = null;
- protected HashMap<Long,ArrayList<ProgramBlock>> _pbcache = null;
-
-
- static
- {
- //init static ID sequence generators
- _pfIDSeq = new IDSequence();
- _pwIDSeq = new IDSequence();
- }
-
- public ParForProgramBlock(Program prog, String[] iterPredVars, HashMap<String,String> params)
- throws DMLRuntimeException
- {
- this( -1, prog, iterPredVars, params);
- }
-
- /**
- * ParForProgramBlock constructor. It reads the specified parameter settings, where defaults for non-specified parameters
- * have been set in ParForStatementBlock.validate(). Furthermore, it generates the IDs for the ParWorkers.
- *
- * @param prog
- * @param iterPred
- * @throws DMLRuntimeException
- */
- public ParForProgramBlock(int ID, Program prog, String[] iterPredVars, HashMap<String,String> params)
- throws DMLRuntimeException
- {
- super(prog, iterPredVars);
-
- //init internal flags according to DML config
- initInternalConfigurations(ConfigurationManager.getConfig());
-
- //ID generation and setting
- setParForProgramBlockIDs( ID );
- _resultVarsIDSeq = new IDSequence();
- _dpVarsIDSeq = new IDSequence();
-
- //parse and use internal parameters (already set to default if not specified)
- _params = params;
- try
- {
- _numThreads = Integer.parseInt( _params.get(ParForStatementBlock.PAR) );
- _taskPartitioner = PTaskPartitioner.valueOf( _params.get(ParForStatementBlock.TASK_PARTITIONER).toUpperCase() );
- _taskSize = Integer.parseInt( _params.get(ParForStatementBlock.TASK_SIZE) );
- _dataPartitioner = PDataPartitioner.valueOf( _params.get(ParForStatementBlock.DATA_PARTITIONER).toUpperCase() );
- _resultMerge = PResultMerge.valueOf( _params.get(ParForStatementBlock.RESULT_MERGE).toUpperCase() );
- _execMode = PExecMode.valueOf( _params.get(ParForStatementBlock.EXEC_MODE).toUpperCase() );
- _optMode = POptMode.valueOf( _params.get(ParForStatementBlock.OPT_MODE).toUpperCase());
- _optLogLevel = Level.toLevel( _params.get(ParForStatementBlock.OPT_LOG));
- _monitor = (Integer.parseInt(_params.get(ParForStatementBlock.PROFILE) ) == 1);
- }
- catch(Exception ex)
- {
- //runtime exception in order to keep signature of program block
- throw new RuntimeException("Error parsing specified ParFOR parameters.",ex);
- }
-
- //reset the internal opt mode if optimization globally disabled.
- if( !OPTIMIZE )
- _optMode = POptMode.NONE;
-
- _variablesDPOriginal = new LocalVariableMap();
- _variablesDPReuse = new LocalVariableMap();
-
- //create IDs for all parworkers
- if( _execMode == PExecMode.LOCAL /*&& _optMode==POptMode.NONE*/ )
- setLocalParWorkerIDs();
-
- //initialize program block cache if necessary
- if( USE_PB_CACHE )
- _pbcache = new HashMap<Long, ArrayList<ProgramBlock>>();
-
- //created profiling report after parfor exec
- _monitorReport = _monitor;
-
- //materialized meta data (reused for all invocations)
- _hasFunctions = ProgramRecompiler.containsAtLeastOneFunction(this);
-
- LOG.trace("PARFOR: ParForProgramBlock created with mode = "+_execMode+", optmode = "+_optMode+", numThreads = "+_numThreads);
- }
-
- public long getID()
- {
- return _ID;
- }
-
- public PExecMode getExecMode()
- {
- return _execMode;
- }
-
- public HashMap<String,String> getParForParams()
- {
- return _params;
- }
-
- public ArrayList<String> getResultVariables()
- {
- return _resultVars;
- }
-
- public void setResultVariables(ArrayList<String> resultVars)
- {
- _resultVars = resultVars;
- }
-
- public void disableOptimization()
- {
- _optMode = POptMode.NONE;
- }
-
- public POptMode getOptimizationMode()
- {
- return _optMode;
- }
-
- public int getDegreeOfParallelism()
- {
- return _numThreads;
- }
-
- public void setDegreeOfParallelism(int k)
- {
- _numThreads = k;
- _params.put(ParForStatementBlock.PAR, String.valueOf(_numThreads)); //kept up-to-date for copies
- setLocalParWorkerIDs();
- }
-
- public void setCPCaching(boolean flag)
- {
- _enableCPCaching = flag;
- }
-
- public void setRuntimePiggybacking(boolean flag)
- {
- _enableRuntimePiggybacking = flag;
- }
-
- public void setExecMode( PExecMode mode )
- {
- _execMode = mode;
- _params.put(ParForStatementBlock.EXEC_MODE, String.valueOf(_execMode)); //kept up-to-date for copies
- }
-
- public void setTaskPartitioner( PTaskPartitioner partitioner )
- {
- _taskPartitioner = partitioner;
- _params.put(ParForStatementBlock.TASK_PARTITIONER, String.valueOf(_taskPartitioner)); //kept up-to-date for copies
- }
-
- public void setTaskSize( long tasksize )
- {
- _taskSize = tasksize;
- _params.put(ParForStatementBlock.TASK_SIZE, String.valueOf(_taskSize)); //kept up-to-date for copies
- }
-
- public void setDataPartitioner(PDataPartitioner partitioner)
- {
- _dataPartitioner = partitioner;
- _params.put(ParForStatementBlock.DATA_PARTITIONER, String.valueOf(_dataPartitioner)); //kept up-to-date for copies
- }
-
- public void enableColocatedPartitionedMatrix( String varname )
- {
- //only called from optimizer
- _colocatedDPMatrix = varname;
- }
-
- public void setTransposeSparseColumnVector( boolean flag )
- {
- _tSparseCol = flag;
- }
-
- public void setPartitionReplicationFactor( int rep )
- {
- //only called from optimizer
- _replicationDP = rep;
- }
-
- public void setExportReplicationFactor( int rep )
- {
- //only called from optimizer
- _replicationExport = rep;
- }
-
- public void disableJVMReuse()
- {
- //only called from optimizer
- _jvmReuse = false;
- }
-
- public void disableMonitorReport()
- {
- _monitorReport = false;
- }
-
- public void setResultMerge(PResultMerge merge)
- {
- _resultMerge = merge;
- _params.put(ParForStatementBlock.RESULT_MERGE, String.valueOf(_resultMerge)); //kept up-to-date for copies
- }
-
- public void setRecompileMemoryBudget( double localMem )
- {
- _recompileMemoryBudget = localMem;
- }
-
- public void setSparkRepartitionVariables(Collection<String> vars) {
- _variablesRP = vars;
- }
-
- public Collection<String> getSparkRepartitionVariables() {
- return _variablesRP;
- }
-
- public void setSparkEagerCacheVariables(Collection<String> vars) {
- _variablesECache = vars;
- }
-
- public long getNumIterations()
- {
- return _numIterations;
- }
-
- public boolean hasFunctions() {
- return _hasFunctions;
- }
-
- public static void initInternalConfigurations( DMLConfig conf )
- {
- ALLOW_REUSE_MR_JVMS = conf.getBooleanValue(DMLConfig.JVM_REUSE);
- ALLOW_REUSE_MR_PAR_WORKER = ALLOW_REUSE_MR_JVMS;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
-
- // add the iterable predicate variable to the variable set
- String iterVarName = _iterablePredicateVars[0];
-
- // evaluate from, to, incr only once (assumption: known at for entry)
- IntObject from = executePredicateInstructions( 1, _fromInstructions, ec );
- IntObject to = executePredicateInstructions( 2, _toInstructions, ec );
- IntObject incr = executePredicateInstructions( 3, _incrementInstructions, ec );
-
- if ( incr.getLongValue() <= 0 ) //would produce infinite loop
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment of variable '" + iterVarName + "' must evaluate to a positive value.");
-
- //early exit on num iterations = zero
- if( computeNumIterations(from, to, incr) <= 0 )
- return; //avoid unnecessary optimization/initialization
-
- ///////
- //OPTIMIZATION of ParFOR body (incl all child parfor PBs)
- ///////
- if( _optMode != POptMode.NONE )
- {
- updateIterablePredicateVars( iterVarName, from, to, incr );
- OptimizationWrapper.setLogLevel(_optLogLevel); //set optimizer log level
- OptimizationWrapper.optimize( _optMode, sb, this, ec, _monitor ); //core optimize
-
- //take changed iterable predicate into account
- iterVarName = _iterablePredicateVars[0];
- from = executePredicateInstructions( 1, _fromInstructions, ec );
- to = executePredicateInstructions( 2, _toInstructions, ec );
- incr = executePredicateInstructions( 3, _incrementInstructions, ec );
- }
-
- ///////
- //DATA PARTITIONING of read-only parent variables of type (matrix,unpartitioned)
- ///////
- Timing time = _monitor ? new Timing(true) : null;
-
- //partitioning on demand (note: for fused data partitioning and execute the optimizer set
- //the data partitioner to NONE in order to prevent any side effects)
- handleDataPartitioning( ec );
-
- //repartitioning of variables for spark cpmm/zipmm in order prevent unnecessary shuffle
- handleSparkRepartitioning( ec );
-
- //eager rdd caching of variables for spark in order prevent read/write contention
- handleSparkEagerCaching( ec );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_DATA_T, time.stop());
-
- // initialize iter var to form value
- IntObject iterVar = new IntObject(iterVarName, from.getLongValue() );
-
- ///////
- //begin PARALLEL EXECUTION of (PAR)FOR body
- ///////
- LOG.trace("EXECUTE PARFOR ID = "+_ID+" with mode = "+_execMode+", numThreads = "+_numThreads+", taskpartitioner = "+_taskPartitioner);
-
- if( _monitor )
- {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTHREADS, _numThreads);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_TASKSIZE, _taskSize);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_TASKPARTITIONER, _taskPartitioner.ordinal());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_DATAPARTITIONER, _dataPartitioner.ordinal());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_EXECMODE, _execMode.ordinal());
- }
-
- //preserve shared input/result variables of cleanup
- ArrayList<String> varList = ec.getVarList();
- HashMap<String, Boolean> varState = ec.pinVariables(varList);
-
- try
- {
- switch( _execMode )
- {
- case LOCAL: //create parworkers as local threads
- executeLocalParFor(ec, iterVar, from, to, incr);
- break;
-
- case REMOTE_MR: // create parworkers as MR tasks (one job per parfor)
- executeRemoteMRParFor(ec, iterVar, from, to, incr);
- break;
-
- case REMOTE_MR_DP: // create parworkers as MR tasks (one job per parfor)
- executeRemoteMRParForDP(ec, iterVar, from, to, incr);
- break;
-
- case REMOTE_SPARK: // create parworkers as Spark tasks (one job per parfor)
- executeRemoteSparkParFor(ec, iterVar, from, to, incr);
- break;
-
- case REMOTE_SPARK_DP: // create parworkers as Spark tasks (one job per parfor)
- executeRemoteSparkParForDP(ec, iterVar, from, to, incr);
- break;
-
- default:
- throw new DMLRuntimeException("Undefined execution mode: '"+_execMode+"'.");
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("PARFOR: Failed to execute loop in parallel.",ex);
- }
-
- //reset state of shared input/result variables
- ec.unpinVariables(varList, varState);
-
- //cleanup unpinned shared variables
- cleanupSharedVariables(ec, varState);
-
- //set iteration var to TO value (+ increment) for FOR equivalence
- iterVar = new IntObject( iterVarName, to.getLongValue() ); //consistent with for
- ec.setVariable(iterVarName, iterVar);
-
- //ensure that subsequent program blocks never see partitioned data (invalid plans!)
- //we can replace those variables, because partitioning only applied for read-only matrices
- for( String var : _variablesDPOriginal.keySet() )
- {
- //cleanup partitioned matrix (if not reused)
- if( !_variablesDPReuse.keySet().contains(var) )
- VariableCPInstruction.processRemoveVariableInstruction(ec, var);
- //reset to original matrix
- MatrixObject mo = (MatrixObject) _variablesDPOriginal.get( var );
- ec.setVariable(var, mo);
- }
-
- ///////
- //end PARALLEL EXECUTION of (PAR)FOR body
- ///////
-
- //print profiling report (only if top-level parfor because otherwise in parallel context)
- if( _monitorReport )
- LOG.info("\n"+StatisticMonitor.createReport());
-
- //reset flags/modifications made by optimizer
- //TODO reset of hop parallelism constraint (e.g., ba+*)
- for( String dpvar : _variablesDPOriginal.keySet() ) //release forced exectypes
- ProgramRecompiler.rFindAndRecompileIndexingHOP(sb, this, dpvar, ec, false);
- //release forced exectypes for fused dp/exec
- if( _execMode == PExecMode.REMOTE_MR_DP || _execMode == PExecMode.REMOTE_SPARK_DP )
- ProgramRecompiler.rFindAndRecompileIndexingHOP(sb, this, _colocatedDPMatrix, ec, false);
- resetIterablePredicateVars();
- resetOptimizerFlags(); //after release, deletes dp_varnames
-
- //execute exit instructions (usually empty)
- executeInstructions(_exitInstructions, ec);
- }
-
-
- /**
- * Executes the parfor locally, i.e., the parfor is realized with numThreads local threads that drive execution.
- * This execution mode allows for arbitrary nested local parallelism and nested invocations of MR jobs. See
- * below for details of the realization.
- *
- * @param ec
- * @param itervar
- * @param from
- * @param to
- * @param incr
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws InterruptedException
- */
- private void executeLocalParFor( ExecutionContext ec, IntObject itervar, IntObject from, IntObject to, IntObject incr )
- throws DMLUnsupportedOperationException, DMLRuntimeException, InterruptedException
- {
- /* Step 1) init parallel workers, task queue and threads
- * start threads (from now on waiting for tasks)
- * Step 2) create tasks
- * put tasks into queue
- * mark end of task input stream
- * Step 3) join all threads (wait for finished work)
- * Step 4) collect results from each parallel worker
- */
-
- Timing time = new Timing(true);
-
- int numExecutedTasks = 0;
- int numExecutedIterations = 0;
-
- //restrict recompilation to thread local memory
- setMemoryBudget();
-
- //enable runtime piggybacking if required
- if( _enableRuntimePiggybacking )
- RuntimePiggybacking.start( _numThreads ); //default piggybacking worker
-
- try
- {
- // Step 1) init parallel workers, task queue and threads
- LocalTaskQueue<Task> queue = new LocalTaskQueue<Task>();
- Thread[] threads = new Thread[_numThreads];
- LocalParWorker[] workers = new LocalParWorker[_numThreads];
- for( int i=0; i<_numThreads; i++ )
- {
- //create parallel workers as (lazy) deep copies
- workers[i] = createParallelWorker( _pwIDs[i], queue, ec );
- threads[i] = new Thread( workers[i] );
- threads[i].setPriority(Thread.MAX_PRIORITY);
- }
-
- // start threads (from now on waiting for tasks)
- for( Thread thread : threads )
- thread.start();
-
- //maintain statistics
- long tinit = (long) time.stop();
- if( DMLScript.STATISTICS )
- Statistics.incrementParForInitTime(tinit);
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_PARWRK_T, tinit);
-
- // Step 2) create tasks
- TaskPartitioner partitioner = createTaskPartitioner(from, to, incr);
- long numIterations = partitioner.getNumIterations();
- long numCreatedTasks = -1;
- if( USE_STREAMING_TASK_CREATION )
- {
- //put tasks into queue (parworker start work on first tasks while creating tasks)
- numCreatedTasks = partitioner.createTasks(queue);
- }
- else
- {
- List<Task> tasks = partitioner.createTasks();
- numCreatedTasks = tasks.size();
-
- // put tasks into queue
- for( Task t : tasks )
- queue.enqueueTask( t );
-
- // mark end of task input stream
- queue.closeInput();
- }
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop());
-
- // Step 3) join all threads (wait for finished work)
- for( Thread thread : threads )
- thread.join();
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_EXEC_T, time.stop());
-
-
- // Step 4) collecting results from each parallel worker
- //obtain results
- LocalVariableMap [] localVariables = new LocalVariableMap [_numThreads];
- for( int i=0; i<_numThreads; i++ )
- {
- localVariables[i] = workers[i].getVariables();
- numExecutedTasks += workers[i].getExecutedTasks();
- numExecutedIterations += workers[i].getExecutedIterations();
- }
- //consolidate results into global symbol table
- consolidateAndCheckResults( ec, numIterations, numCreatedTasks, numExecutedIterations, numExecutedTasks,
- localVariables );
-
- // Step 5) cleanup local parworkers (e.g., remove created functions)
- for( int i=0; i<_numThreads; i++ )
- {
- Collection<String> fnNames = workers[i].getFunctionNames();
- if( fnNames!=null && !fnNames.isEmpty() )
- for( String fn : fnNames ) {
- String[] parts = DMLProgram.splitFunctionKey(fn);
- _prog.removeFunctionProgramBlock(parts[0], parts[1]);
- }
- }
- }
- finally
- {
- //remove thread-local memory budget (reset to original budget)
- //(in finally to prevent error side effects for multiple scripts in one jvm)
- resetMemoryBudget();
-
- //disable runtime piggybacking
- if( _enableRuntimePiggybacking )
- RuntimePiggybacking.stop();
-
- if( _monitor ) {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_RESULTS_T, time.stop());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTASKS, numExecutedTasks);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMITERS, numExecutedIterations);
- }
- }
- }
-
- /**
- *
- * @param ec
- * @param itervar
- * @param from
- * @param to
- * @param incr
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void executeRemoteMRParFor( ExecutionContext ec, IntObject itervar, IntObject from, IntObject to, IntObject incr )
- throws DMLUnsupportedOperationException, DMLRuntimeException, IOException
- {
- /* Step 0) check and recompile MR inst
- * Step 1) serialize child PB and inst
- * Step 2) create tasks
- * serialize tasks
- * Step 3) submit MR Jobs and wait for results
- * Step 4) collect results from each parallel worker
- */
-
- Timing time = ( _monitor ? new Timing(true) : null );
-
- // Step 0) check and compile to CP (if forced remote parfor)
- boolean flagForced = false;
- if( FORCE_CP_ON_REMOTE_MR && (_optMode == POptMode.NONE || (_optMode == POptMode.CONSTRAINED && _execMode==PExecMode.REMOTE_MR)) )
- {
- //tid = 0 because replaced in remote parworker
- flagForced = checkMRAndRecompileToCP(0);
- }
-
- // Step 1) init parallel workers (serialize PBs)
- // NOTES: each mapper changes filenames with regard to his ID as we submit a single job,
- // cannot reuse serialized string, since variables are serialized as well.
- ParForBody body = new ParForBody( _childBlocks, _resultVars, ec );
- String program = ProgramConverter.serializeParForBody( body );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_PARWRK_T, time.stop());
-
- // Step 2) create tasks
- TaskPartitioner partitioner = createTaskPartitioner(from, to, incr);
- String taskFile = constructTaskFileName();
- String resultFile = constructResultFileName();
-
- long numIterations = partitioner.getNumIterations();
- int maxDigits = (int)Math.log10(to.getLongValue()) + 1;
- long numCreatedTasks = -1;
- if( USE_STREAMING_TASK_CREATION )
- {
- LocalTaskQueue<Task> queue = new LocalTaskQueue<Task>();
-
- //put tasks into queue and start writing to taskFile
- numCreatedTasks = partitioner.createTasks(queue);
- taskFile = writeTasksToFile( taskFile, queue, maxDigits );
- }
- else
- {
- //sequentially create tasks and write to disk
- List<Task> tasks = partitioner.createTasks();
- numCreatedTasks = tasks.size();
- taskFile = writeTasksToFile( taskFile, tasks, maxDigits );
- }
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop());
-
- //write matrices to HDFS
- exportMatricesToHDFS(ec);
-
- // Step 3) submit MR job (wait for finished work)
- MatrixObject colocatedDPMatrixObj = (_colocatedDPMatrix!=null)? (MatrixObject)ec.getVariable(_colocatedDPMatrix) : null;
- RemoteParForJobReturn ret = RemoteParForMR.runJob(_ID, program, taskFile, resultFile, colocatedDPMatrixObj, _enableCPCaching,
- _numThreads, WRITE_REPLICATION_FACTOR, MAX_RETRYS_ON_ERROR, getMinMemory(ec),
- (ALLOW_REUSE_MR_JVMS & _jvmReuse) );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_EXEC_T, time.stop());
-
-
- // Step 4) collecting results from each parallel worker
- int numExecutedTasks = ret.getNumExecutedTasks();
- int numExecutedIterations = ret.getNumExecutedIterations();
-
- //consolidate results into global symbol table
- consolidateAndCheckResults( ec, numIterations, numCreatedTasks, numExecutedIterations , numExecutedTasks,
- ret.getVariables() );
- if( flagForced ) //see step 0
- releaseForcedRecompile(0);
-
- if( _monitor )
- {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_RESULTS_T, time.stop());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTASKS, numExecutedTasks);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMITERS, numExecutedIterations);
- }
- }
-
- /**
- *
- * @param ec
- * @param itervar
- * @param from
- * @param to
- * @param incr
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void executeRemoteMRParForDP( ExecutionContext ec, IntObject itervar, IntObject from, IntObject to, IntObject incr )
- throws DMLUnsupportedOperationException, DMLRuntimeException, IOException
- {
- /* Step 0) check and recompile MR inst
- * Step 1) serialize child PB and inst
- * Step 2) create tasks
- * serialize tasks
- * Step 3) submit MR Jobs and wait for results
- * Step 4) collect results from each parallel worker
- */
-
- Timing time = ( _monitor ? new Timing(true) : null );
-
- // Step 0) check and compile to CP (if forced remote parfor)
- boolean flagForced = checkMRAndRecompileToCP(0);
-
- // Step 1) prepare partitioned input matrix (needs to happen before serializing the progam)
- ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
- MatrixObject inputMatrix = (MatrixObject)ec.getVariable(_colocatedDPMatrix );
- PDataPartitionFormat inputDPF = sb.determineDataPartitionFormat( _colocatedDPMatrix );
- inputMatrix.setPartitioned(inputDPF, 1); //mark matrix var as partitioned (for reducers)
-
- // Step 2) init parallel workers (serialize PBs)
- // NOTES: each mapper changes filenames with regard to his ID as we submit a single job,
- // cannot reuse serialized string, since variables are serialized as well.
- ParForBody body = new ParForBody( _childBlocks, _resultVars, ec );
- String program = ProgramConverter.serializeParForBody( body );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_PARWRK_T, time.stop());
-
- // Step 3) create tasks
- TaskPartitioner partitioner = createTaskPartitioner(from, to, incr);
- String resultFile = constructResultFileName();
- long numIterations = partitioner.getNumIterations();
- long numCreatedTasks = numIterations;//partitioner.createTasks().size();
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop());
-
- //write matrices to HDFS
- exportMatricesToHDFS(ec);
-
- // Step 4) submit MR job (wait for finished work)
- OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
- (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))?
- OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
- RemoteParForJobReturn ret = RemoteDPParForMR.runJob(_ID, itervar.getName(), _colocatedDPMatrix, program, resultFile,
- inputMatrix, inputDPF, inputOI, _tSparseCol, _enableCPCaching, _numThreads, _replicationDP, MAX_RETRYS_ON_ERROR );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_EXEC_T, time.stop());
-
- // Step 5) collecting results from each parallel worker
- int numExecutedTasks = ret.getNumExecutedTasks();
- int numExecutedIterations = ret.getNumExecutedIterations();
-
- //consolidate results into global symbol table
- consolidateAndCheckResults( ec, numIterations, numCreatedTasks, numExecutedIterations, numExecutedTasks,
- ret.getVariables() );
-
- if( flagForced ) //see step 0
- releaseForcedRecompile(0);
- inputMatrix.unsetPartitioned();
-
- if( _monitor )
- {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_RESULTS_T, time.stop());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTASKS, numExecutedTasks);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMITERS, numExecutedIterations);
- }
- }
-
- /**
- *
- * @param ec
- * @param itervar
- * @param from
- * @param to
- * @param incr
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void executeRemoteSparkParFor(ExecutionContext ec, IntObject itervar, IntObject from, IntObject to, IntObject incr)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- Timing time = ( _monitor ? new Timing(true) : null );
-
- // Step 0) check and compile to CP (if forced remote parfor)
- boolean flagForced = false;
- if( FORCE_CP_ON_REMOTE_MR && (_optMode == POptMode.NONE || (_optMode == POptMode.CONSTRAINED && _execMode==PExecMode.REMOTE_SPARK)) )
- {
- //tid = 0 because replaced in remote parworker
- flagForced = checkMRAndRecompileToCP(0);
- }
-
- // Step 1) init parallel workers (serialize PBs)
- // NOTES: each mapper changes filenames with regard to his ID as we submit a single job,
- // cannot reuse serialized string, since variables are serialized as well.
- ParForBody body = new ParForBody( _childBlocks, _resultVars, ec );
- String program = ProgramConverter.serializeParForBody( body );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_PARWRK_T, time.stop());
-
- // Step 2) create tasks
- TaskPartitioner partitioner = createTaskPartitioner(from, to, incr);
- long numIterations = partitioner.getNumIterations();
-
- //sequentially create tasks as input to parfor job
- List<Task> tasks = partitioner.createTasks();
- long numCreatedTasks = tasks.size();
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop());
-
- //write matrices to HDFS
- exportMatricesToHDFS(ec);
-
- // Step 3) submit Spark parfor job (no lazy evaluation, since collect on result)
- //MatrixObject colocatedDPMatrixObj = (_colocatedDPMatrix!=null)? (MatrixObject)ec.getVariable(_colocatedDPMatrix) : null;
- RemoteParForJobReturn ret = RemoteParForSpark.runJob(_ID, program, tasks, ec, _enableCPCaching, _numThreads);
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_EXEC_T, time.stop());
-
-
- // Step 4) collecting results from each parallel worker
- int numExecutedTasks = ret.getNumExecutedTasks();
- int numExecutedIterations = ret.getNumExecutedIterations();
-
- //consolidate results into global symbol table
- consolidateAndCheckResults( ec, numIterations, numCreatedTasks, numExecutedIterations , numExecutedTasks,
- ret.getVariables() );
- if( flagForced ) //see step 0
- releaseForcedRecompile(0);
-
- if( _monitor )
- {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_RESULTS_T, time.stop());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTASKS, numExecutedTasks);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMITERS, numExecutedIterations);
- }
- }
-
- private void executeRemoteSparkParForDP( ExecutionContext ec, IntObject itervar, IntObject from, IntObject to, IntObject incr )
- throws DMLUnsupportedOperationException, DMLRuntimeException, IOException
- {
- Timing time = ( _monitor ? new Timing(true) : null );
-
- // Step 0) check and compile to CP (if forced remote parfor)
- boolean flagForced = checkMRAndRecompileToCP(0);
-
- // Step 1) prepare partitioned input matrix (needs to happen before serializing the progam)
- ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
- MatrixObject inputMatrix = (MatrixObject)ec.getVariable(_colocatedDPMatrix );
- PDataPartitionFormat inputDPF = sb.determineDataPartitionFormat( _colocatedDPMatrix );
- inputMatrix.setPartitioned(inputDPF, 1); //mark matrix var as partitioned (for reducers)
-
- // Step 2) init parallel workers (serialize PBs)
- // NOTES: each mapper changes filenames with regard to his ID as we submit a single job,
- // cannot reuse serialized string, since variables are serialized as well.
- ParForBody body = new ParForBody( _childBlocks, _resultVars, ec );
- String program = ProgramConverter.serializeParForBody( body );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_PARWRK_T, time.stop());
-
- // Step 3) create tasks
- TaskPartitioner partitioner = createTaskPartitioner(from, to, incr);
- String resultFile = constructResultFileName();
- long numIterations = partitioner.getNumIterations();
- long numCreatedTasks = numIterations;//partitioner.createTasks().size();
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop());
-
- //write matrices to HDFS
- exportMatricesToHDFS(ec);
-
- // Step 4) submit MR job (wait for finished work)
- OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
- (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))?
- OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
- RemoteParForJobReturn ret = RemoteDPParForSpark.runJob(_ID, itervar.getName(), _colocatedDPMatrix, program, resultFile,
- inputMatrix, ec, inputDPF, inputOI, _tSparseCol, _enableCPCaching, _numThreads );
-
- if( _monitor )
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_EXEC_T, time.stop());
-
- // Step 5) collecting results from each parallel worker
- int numExecutedTasks = ret.getNumExecutedTasks();
- int numExecutedIterations = ret.getNumExecutedIterations();
-
- //consolidate results into global symbol table
- consolidateAndCheckResults( ec, numIterations, numCreatedTasks, numExecutedIterations, numExecutedTasks,
- ret.getVariables() );
-
- if( flagForced ) //see step 0
- releaseForcedRecompile(0);
- inputMatrix.unsetPartitioned();
-
- if( _monitor )
- {
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_WAIT_RESULTS_T, time.stop());
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMTASKS, numExecutedTasks);
- StatisticMonitor.putPFStat(_ID, Stat.PARFOR_NUMITERS, numExecutedIterations);
- }
- }
-
- /**
- *
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void handleDataPartitioning( ExecutionContext ec )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- if( _dataPartitioner != PDataPartitioner.NONE )
- {
- ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
- if( sb == null )
- throw new DMLRuntimeException("ParFor statement block required for reasoning about data partitioning.");
-
- ArrayList<String> vars = sb.getReadOnlyParentVars();
- for( String var : vars )
- {
- Data dat = ec.getVariable(var);
- //skip non-existing input matrices (which are due to unknown sizes marked for
- //partitioning but typically related branches are never executed)
- if( dat != null && dat instanceof MatrixObject )
- {
- MatrixObject moVar = (MatrixObject) dat; //unpartitioned input
-
- PDataPartitionFormat dpf = sb.determineDataPartitionFormat( var );
- //dpf = (_optMode != POptMode.NONE) ? OptimizerRuleBased.decideBlockWisePartitioning(moVar, dpf) : dpf;
- LOG.trace("PARFOR ID = "+_ID+", Partitioning read-only input variable "+var+" (format="+dpf+", mode="+_dataPartitioner+")");
-
- if( dpf != PDataPartitionFormat.NONE )
- {
- Timing ltime = new Timing(true);
-
- //input data partitioning (reuse if possible)
- Data dpdatNew = _variablesDPReuse.get(var);
- if( dpdatNew == null ) //no reuse opportunity
- {
- DataPartitioner dp = createDataPartitioner( dpf, _dataPartitioner, ec );
- //disable binary cell for sparse if consumed by MR jobs
- if( !OptimizerRuleBased.allowsBinaryCellPartitions(moVar, dpf )
- || OptimizerUtils.isSparkExecutionMode() ) //TODO support for binarycell
- {
- dp.disableBinaryCell();
- }
- MatrixObject moVarNew = dp.createPartitionedMatrixObject(moVar, constructDataPartitionsFileName());
- dpdatNew = moVarNew;
-
- //skip remaining partitioning logic if not partitioned (e.g., too small)
- if( moVar == moVarNew )
- continue; //skip to next
- }
- ec.setVariable(var, dpdatNew);
-
- //recompile parfor body program
- ProgramRecompiler.rFindAndRecompileIndexingHOP(sb,this,var,ec,true);
-
- //store original and partitioned matrix (for reuse if applicable)
- _variablesDPOriginal.put(var, moVar);
- if( ALLOW_REUSE_PARTITION_VARS
- && ProgramRecompiler.isApplicableForReuseVariable(sb.getDMLProg(), sb, var) )
- {
- _variablesDPReuse.put(var, dpdatNew);
- }
-
- LOG.trace("Partitioning and recompilation done in "+ltime.stop()+"ms");
- }
- }
- }
- }
- }
-
- /**
- *
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void handleSparkRepartitioning( ExecutionContext ec )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- if( OptimizerUtils.isSparkExecutionMode() &&
- _variablesRP != null && !_variablesRP.isEmpty() ) {
- SparkExecutionContext sec = (SparkExecutionContext) ec;
-
- for( String var : _variablesRP )
- sec.repartitionAndCacheMatrixObject(var);
- }
- }
-
- /**
- *
- * @param ec
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private void handleSparkEagerCaching( ExecutionContext ec )
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- if( OptimizerUtils.isSparkExecutionMode() &&
- _variablesECache != null && !_variablesECache.isEmpty() ) {
- SparkExecutionContext sec = (SparkExecutionContext) ec;
-
- for( String var : _variablesECache )
- sec.cacheMatrixObject(var);
- }
- }
-
- /**
- * Cleanup result variables of parallel workers after result merge.
- * @param in
- * @param out
- * @throws DMLRuntimeException
- */
- private void cleanWorkerResultVariables(ExecutionContext ec, MatrixObject out, MatrixObject[] in)
- throws DMLRuntimeException
- {
- for( MatrixObject tmp : in )
- {
- //check for empty inputs (no iterations executed)
- if( tmp != null && tmp != out )
- ec.cleanupMatrixObject(tmp);
- }
- }
-
- /**
- * Create empty matrix objects and scalars for all unscoped vars
- * (created within the parfor).
- *
- * NOTE: parfor gives no guarantees on the values of those objects - hence
- * we return -1 for sclars and empty matrix objects.
- *
- * @param out
- * @param sb
- * @throws DMLRuntimeException
- */
- private void createEmptyUnscopedVariables( LocalVariableMap out, StatementBlock sb )
- throws DMLRuntimeException
- {
- VariableSet updated = sb.variablesUpdated();
- VariableSet livein = sb.liveIn();
-
- //for all vars IN <updated> AND NOT IN <livein>
- for( String var : updated.getVariableNames() )
- if( !livein.containsVariable(var) )
- {
- //create empty output
- DataIdentifier dat = updated.getVariable(var);
- DataType datatype = dat.getDataType();
- ValueType valuetype = dat.getValueType();
- Data dataObj = null;
- switch( datatype )
- {
- case SCALAR:
- switch( valuetype )
- {
- case BOOLEAN: dataObj = new BooleanObject(var,false); break;
- case INT: dataObj = new IntObject(var,-1); break;
- case DOUBLE: dataObj = new DoubleObject(var,-1d); break;
- case STRING: dataObj = new StringObject(var,"-1"); break;
- default:
- throw new DMLRuntimeException("Value type not supported: "+valuetype);
- }
- break;
- case MATRIX:
- //currently we do not create any unscoped matrix object outputs
- //because metadata (e.g., outputinfo) not known at this place.
- break;
- case UNKNOWN:
- break;
- default:
- throw new DMLRuntimeException("Data type not supported: "+datatype);
- }
-
- if( dataObj != null )
- out.put(var, dataObj);
- }
- }
-
- /**
- *
- * @throws CacheException
- */
- private void exportMatricesToHDFS( ExecutionContext ec )
- throws CacheException
- {
- ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
-
- if( LIVEVAR_AWARE_EXPORT && sb != null)
- {
- //optimization to prevent unnecessary export of matrices
- //export only variables that are read in the body
- VariableSet varsRead = sb.variablesRead();
- for (String key : ec.getVariables().keySet() )
- {
- Data d = ec.getVariable(key);
- if ( d.getDataType() == DataType.MATRIX
- && varsRead.containsVariable(key) )
- {
- MatrixObject mo = (MatrixObject)d;
- mo.exportData( _replicationExport );
- }
- }
- }
- else
- {
- //export all matrices in symbol table
- for (String key : ec.getVariables().keySet() )
- {
- Data d = ec.getVariable(key);
- if ( d.getDataType() == DataType.MATRIX )
- {
- MatrixObject mo = (MatrixObject)d;
- mo.exportData( _replicationExport );
- }
- }
- }
- }
-
- /**
- *
- * @param ec
- * @param varState
- * @throws DMLRuntimeException
- */
- private void cleanupSharedVariables( ExecutionContext ec, HashMap<String,Boolean> varState )
- throws DMLRuntimeException
- {
- //TODO needs as precondition a systematic treatment of persistent read information.
- /*
- if( LIVEVAR_AWARE_CLEANUP && _sb != null)
- {
- //cleanup shared variables after they are unpinned
- VariableSet liveout = _sb.liveOut();
- for( Entry<String, Boolean> var : varState.entrySet() )
- {
- String varname = var.getKey();
- boolean unpinned = var.getValue();
- String fprefix = ConfigurationManager.getConfig().getTextValue("scratch")
- + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID();
-
- //delete unpinned vars if not in liveout (similar like rmvar) and not persistent input
- if( unpinned && !liveout.containsVariable(varname) )
-
- {
- VariableCPInstruction.processRemoveVariableInstruction(ec,varname);
- }
- }
- }
- */
- }
-
- /**
- * Creates a new or partially recycled instance of a parallel worker. Therefore the symbol table, and child
- * program blocks are deep copied. Note that entries of the symbol table are not deep copied because they are replaced
- * anyway on the next write. In case of recycling the deep copies of program blocks are recycled from previous
- * executions of this parfor.
- *
- *
- * @param pwID
- * @param queue
- * @param ec
- * @return
- * @throws InstantiationException
- * @throws IllegalAccessException
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- * @throws CloneNotSupportedException
- */
- private LocalParWorker createParallelWorker(long pwID, LocalTaskQueue<Task> queue, ExecutionContext ec)
- throws DMLRuntimeException
- {
- LocalParWorker pw = null;
-
- try
- {
- //create deep copies of required elements
- //child blocks
- ArrayList<ProgramBlock> cpChildBlocks = null;
- HashSet<String> fnNames = new HashSet<String>();
- if( USE_PB_CACHE )
- {
- if( _pbcache.containsKey(pwID) )
- {
- cpChildBlocks = _pbcache.get(pwID);
- }
- else
- {
- cpChildBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(_childBlocks, pwID, _IDPrefix, new HashSet<String>(), fnNames, false, false);
- _pbcache.put(pwID, cpChildBlocks);
- }
- }
- else
- {
- cpChildBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(_childBlocks, pwID, _IDPrefix, new HashSet<String>(), fnNames, false, false);
- }
-
- // Deep copy Execution Context
- ExecutionContext cpEc = ProgramConverter.createDeepCopyExecutionContext(ec);
-
- //create the actual parallel worker
- ParForBody body = new ParForBody( cpChildBlocks, _resultVars, cpEc );
- pw = new LocalParWorker( pwID, queue, body, MAX_RETRYS_ON_ERROR, _monitor );
- pw.setFunctionNames(fnNames);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- return pw;
- }
-
- /**
- * Creates a new task partitioner according to the specified runtime parameter.
- *
- * @param from
- * @param to
- * @param incr
- * @return
- * @throws DMLRuntimeException
- */
- private TaskPartitioner createTaskPartitioner( IntObject from, IntObject to, IntObject incr )
- throws DMLRuntimeException
- {
- TaskPartitioner tp = null;
-
- switch( _taskPartitioner )
- {
- case FIXED:
- tp = new TaskPartitionerFixedsize( _taskSize, _iterablePredicateVars[0],
- from, to, incr );
- break;
- case NAIVE:
- tp = new TaskPartitionerNaive( _taskSize, _iterablePredicateVars[0],
- from, to, incr );
- break;
- case STATIC:
- tp = new TaskPartitionerStatic( _taskSize, _numThreads, _iterablePredicateVars[0],
- from, to, incr );
- break;
- case FACTORING:
- tp = new TaskPartitionerFactoring( _taskSize,_numThreads, _iterablePredicateVars[0],
- from, to, incr );
- break;
- case FACTORING_CMIN:
- //for constrained factoring the tasksize is used as the minimum constraint
- tp = new TaskPartitionerFactoringCmin( _taskSize,_numThreads, _taskSize, _iterablePredicateVars[0],
- from, to, incr );
- break;
-
- case FACTORING_CMAX:
- //for constrained factoring the tasksize is used as the minimum constraint
- tp = new TaskPartitionerFactoringCmax( _taskSize,_numThreads, _taskSize, _iterablePredicateVars[0],
- from, to, incr );
- break;
- default:
- throw new DMLRuntimeException("Undefined task partitioner: '"+_taskPartitioner+"'.");
- }
-
- return tp;
- }
-
- /**
- * Creates a new data partitioner according to the specified runtime parameter.
- *
- * @param dpf
- * @param dataPartitioner
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- private DataPartitioner createDataPartitioner(PDataPartitionFormat dpf, PDataPartitioner dataPartitioner, ExecutionContext ec)
- throws DMLRuntimeException
- {
- DataPartitioner dp = null;
-
- //determine max degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
- int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() )
- maxNumRed = (int)Math.max( maxNumRed, YarnClusterAnalyzer.getNumCores()/2 );
- int numRed = Math.min(numReducers,maxNumRed);
-
- //create data partitioner
- switch( dataPartitioner )
- {
- case LOCAL:
- dp = new DataPartitionerLocal(dpf, -1, _numThreads);
- break;
- case REMOTE_MR:
- dp = new DataPartitionerRemoteMR( dpf, -1, _ID, numRed,
- _replicationDP,
- MAX_RETRYS_ON_ERROR,
- ALLOW_REUSE_MR_JVMS, false );
- break;
- case REMOTE_SPARK:
- dp = new DataPartitionerRemoteSpark( dpf, -1, ec, numRed, false );
- break;
- default:
- throw new DMLRuntimeException("Undefined data partitioner: '" +dataPartitioner.toString()+"'.");
- }
-
- return dp;
- }
-
- /**
- *
- * @param prm
- * @param out
- * @param in
- * @param fname
- * @return
- * @throws DMLRuntimeException
- */
- private ResultMerge createResultMerge( PResultMerge prm, MatrixObject out, MatrixObject[] in, String fname, ExecutionContext ec )
- throws DMLRuntimeException
- {
- ResultMerge rm = null;
-
- //determine degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
- int maxMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
- int maxRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
- //correction max number of reducers on yarn clusters
- if( InfrastructureAnalyzer.isYarnEnabled() ) {
- maxMap = (int)Math.max( maxMap, YarnClusterAnalyzer.getNumCores() );
- maxRed = (int)Math.max( maxRed, YarnClusterAnalyzer.getNumCores()/2 );
- }
- int numMap = Math.max(_numThreads, maxMap);
- int numRed = Math.min(numReducers, maxRed);
-
- //create result merge implementation
- switch( prm )
- {
- case LOCAL_MEM:
- rm = new ResultMergeLocalMemory( out, in, fname );
- break;
- case LOCAL_FILE:
- rm = new ResultMergeLocalFile( out, in, fname );
- break;
- case LOCAL_AUTOMATIC:
- rm = new ResultMergeLocalAutomatic( out, in, fname );
- break;
- case REMOTE_MR:
- rm = new ResultMergeRemoteMR( out, in, fname, _ID, numMap, numRed,
- WRITE_REPLICATION_FACTOR,
- MAX_RETRYS_ON_ERROR,
- ALLOW_REUSE_MR_JVMS );
- break;
- case REMOTE_SPARK:
- rm = new ResultMergeRemoteSpark( out, in, fname, ec, numMap, numRed );
- break;
-
- default:
- throw new DMLRuntimeException("Undefined result merge: '" +prm.toString()+"'.");
- }
-
- return rm;
- }
-
- /**
- * Recompile program block hierarchy to forced CP if MR instructions or functions.
- * Returns true if recompile was necessary and possible
- *
- * @param tid
- * @return
- * @throws DMLRuntimeException
- */
- private boolean checkMRAndRecompileToCP(long tid)
- throws DMLRuntimeException
- {
- //no MR instructions, ok
- if( !OptTreeConverter.rContainsMRJobInstruction(this, true) )
- return false;
-
- //no statement block, failed
- ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
- if( sb == null ) {
- LOG.warn("Missing parfor statement block for recompile.");
- return false;
- }
-
- //try recompile MR instructions to CP
- HashSet<String> fnStack = new HashSet<String>();
- Recompiler.recompileProgramBlockHierarchy2Forced(_childBlocks, tid, fnStack, ExecType.CP);
- return true;
- }
-
- /**
- *
- * @param tid
- * @throws DMLRuntimeException
- */
- private void releaseForcedRecompile(long tid)
- throws DMLRuntimeException
- {
- HashSet<String> fnStack = new HashSet<String>();
- Recompiler.recompileProgramBlockHierarchy2Forced(_childBlocks, tid, fnStack, null);
- }
-
-
- /**
- *
- * @param fname
- * @param tasks
- * @return
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private String writeTasksToFile(String fname, List<Task> tasks, int maxDigits)
- throws DMLRuntimeException, IOException
- {
- BufferedWriter br = null;
- try
- {
- Path path = new Path(fname);
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
-
- boolean flagFirst = true; //workaround for keeping gen order
- for( Task t : tasks )
- {
- br.write( createTaskFileLine( t, maxDigits, flagFirst ) );
- if( flagFirst )
- flagFirst = false;
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Error writing tasks to taskfile "+fname, ex);
- }
- finally
- {
- if( br !=null )
- br.close();
- }
-
- return fname;
- }
-
- /**
- *
- * @param fname
- * @param queue
- * @return
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private String writeTasksToFile(String fname, LocalTaskQueue<Task> queue, int maxDigits)
- throws DMLRuntimeException, IOException
- {
- BufferedWriter br = null;
- try
- {
- Path path = new Path( fname );
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
-
- Task t = null;
- boolean flagFirst = true; //workaround for keeping gen order
- while( (t = queue.dequeueTask()) != LocalTaskQueue.NO_MORE_TASKS )
- {
- br.write( createTaskFileLine( t, maxDigits, flagFirst ) );
- if( flagFirst )
- flagFirst = false;
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Error writing tasks to taskfile "+fname, ex);
- }
- finally
- {
- if( br !=null )
- br.close();
- }
-
- return fname;
- }
-
- private String createTaskFileLine( Task t, int maxDigits, boolean flagFirst )
- {
- //always pad to max digits in order to preserve task order
- String ret = t.toCompactString(maxDigits) + (flagFirst?" ":"") + "\n";
- return ret;
- }
-
- /**
- *
- * @param expIters
- * @param expTasks
- * @param numIters
- * @param numTasks
- * @param results
- * @throws DMLRuntimeException
- */
- private void consolidateAndCheckResults(ExecutionContext ec, long expIters, long expTasks, long numIters, long numTasks, LocalVariableMap [] results)
- throws DMLRuntimeException
- {
- Timing time = new Timing(true);
-
- //result merge
- if( checkParallelRemoteResultMerge() )
- {
- //execute result merge in parallel for all result vars
- int par = Math.min( _resultVars.size(),
- InfrastructureAnalyzer.getLocalParallelism() );
-
- try
- {
- //enqueue all result vars as tasks
- LocalTaskQueue<String> q = new LocalTaskQueue<String>();
- for( String var : _resultVars ) //foreach non-local write
- if( ec.getVariable(var) instanceof MatrixObject ) //robustness scalars
- q.enqueueTask(var);
- q.closeInput();
-
- //run result merge workers
- Thread[] rmWorkers = new Thread[par];
- for( int i=0; i<par; i++ )
- rmWorkers[i] = new Thread(new ResultMergeWorker(q, results, ec));
- for( int i=0; i<par; i++ ) //start all
- rmWorkers[i].start();
- for( int i=0; i<par; i++ ) //wait for all
- rmWorkers[i].join();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
- else
- {
- //execute result merge sequentially for all result vars
- for( String var : _resultVars ) //foreach non-local write
- {
- Data dat = ec.getVariable(var);
- if( dat instanceof MatrixObject ) //robustness scalars
- {
- MatrixObject out = (MatrixObject) dat;
- MatrixObject[] in = new MatrixObject[ results.length ];
- for( int i=0; i< results.length; i++ )
- in[i] = (MatrixObject) results[i].get( var );
- String fname = constructResultMergeFileName();
- ResultMerge rm = createResultMerge(_resultMerge, out, in, fname, ec);
- MatrixObject outNew = null;
- if( USE_PARALLEL_RESULT_MERGE )
- outNew = rm.executeParallelMerge( _numThreads );
- else
- outNew = rm.executeSerialMerge();
-
- //cleanup existing var
- Data exdata = ec.removeVariable(var);
- if( exdata != null && exdata != outNew && exdata instanceof MatrixObject )
- ec.cleanupMatrixObject((MatrixObject)exdata);
-
- //cleanup of intermediate result variables
- cleanWorkerResultVariables( ec, out, in );
-
- //set merged result variable
- ec.setVariable(var, outNew);
- }
- }
- }
-
- //handle unscoped variables (vars created in parfor, but potentially used afterwards)
- ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
- if( CREATE_UNSCOPED_RESULTVARS && sb != null && ec.getVariables() != null ) //sb might be null for nested parallelism
- createEmptyUnscopedVariables( ec.getVariables(), sb );
-
- //check expected counters
- if( numTasks != expTasks || numIters !=expIters ) //consistency check
- throw new DMLRuntimeException("PARFOR: Number of executed tasks does not match the number of created tasks: tasks "+numTasks+"/"+expTasks+", iters "+numIters+"/"+expIters+".");
-
- if( DMLScript.STATISTICS )
- Statistics.incrementParForMergeTime((long) time.stop());
- }
-
- /**
- * NOTE: Currently we use a fixed rule (multiple results AND REMOTE_MR -> only selected by the optimizer
- * if mode was REMOTE_MR as well).
- *
- * TODO The optimizer should explicitly decide about parallel result merge and its degree of parallelism.
- *
- * @return
- */
- private boolean checkParallelRemoteResultMerge()
- {
- return (USE_PARALLEL_RESULT_MERGE_REMOTE
- && _resultVars.size() > 1
- && ( _resultMerge == PResultMerge.REMOTE_MR
- ||_resultMerge == PResultMerge.REMOTE_SPARK) );
- }
-
- /**
- *
- * @param IDPrefix
- */
- private void setParForProgramBlockIDs(int IDPrefix)
- {
- _IDPrefix = IDPrefix;
- if( _IDPrefix == -1 ) //not specified
- _ID = _pfIDSeq.getNextID(); //generated new ID
- else //remote case (further nested parfors are all in one JVM)
- _ID = IDHandler.concatIntIDsToLong(_IDPrefix, (int)_pfIDSeq.getNextID());
- }
-
- /**
- * TODO rework id handling in order to enable worker reuse
- *
- */
- private void setLocalParWorkerIDs()
- {
- if( _numThreads<=0 )
- return;
-
- //set all parworker IDs required if PExecMode.LOCAL is used
-
- _pwIDs = new long[ _numThreads ];
-
- for( int i=0; i<_numThreads; i++ )
- {
- if(_IDPrefix == -1)
- _pwIDs[i] = _pwIDSeq.getNextID();
- else
- _pwIDs[i] = IDHandler.concatIntIDsToLong(_IDPrefix,(int)_pwIDSeq.getNextID());
-
- if( _monitor )
- StatisticMonitor.putPfPwMapping(_ID, _pwIDs[i]);
- }
- }
-
- /**
- *
- * @param from
- * @param to
- * @param incr
- */
- private long computeNumIterations( IntObject from, IntObject to, IntObject incr )
- {
- return (long)Math.ceil(((double)(to.getLongValue() - from.getLongValue() + 1)) / incr.getLongValue());
- }
-
- /**
- *
- * @param iterVarName
- * @param from
- * @param to
- * @param incr
- */
- private void updateIterablePredicateVars(String iterVarName, IntObject from, IntObject to, IntObject incr)
- {
- _numIterations = computeNumIterations(from, to, incr);
-
- //keep original iterable predicate
- _iterablePredicateVarsOriginal = new String[4];
- System.arraycopy(_iterablePredicateVars, 0, _iterablePredicateVarsOriginal, 0, 4);
-
- _iterablePredicateVars[0] = iterVarName;
- _iterablePredicateVars[1] = from.getStringValue();
- _iterablePredicateVars[2] = to.getStringValue();
- _iterablePredicateVars[3] = incr.getStringValue();
- }
-
- /**
- *
- */
- private void resetIterablePredicateVars()
- {
- //reset of modified for optimization (opt!=NONE)
- if( _iterablePredicateVarsOriginal!=null )
- System.arraycopy(_iterablePredicateVarsOriginal, 0, _iterablePredicateVars, 0, 4);
- }
-
- /**
- * NOTE: Only required for remote parfor. Hence, there is no need to transfer DMLConfig to
- * the remote workers (MR job) since nested remote parfor is not supported.
- *
- * @return
- */
- private String constructTaskFileName()
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
-
- StringBuilder sb = new StringBuilder();
- sb.append(scratchSpaceLoc);
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(Lop.PROCESS_PREFIX);
- sb.append(DMLScript.getUUID());
- sb.append(PARFOR_MR_TASKS_TMP_FNAME.replaceAll("%ID%", String.valueOf(_ID)));
-
- return sb.toString();
- }
-
- /**
- * NOTE: Only required for remote parfor. Hence, there is no need to transfer DMLConfig to
- * the remote workers (MR job) since nested remote parfor is not supported.
- *
- * @return
- */
- private String constructResultFileName()
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
-
- StringBuilder sb = new StringBuilder();
- sb.append(scratchSpaceLoc);
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(Lop.PROCESS_PREFIX);
- sb.append(DMLScript.getUUID());
- sb.append(PARFOR_MR_RESULT_TMP_FNAME.replaceAll("%ID%", String.valueOf(_ID)));
-
- return sb.toString();
- }
-
- /**
- *
- * @return
- */
- private String constructResultMergeFileName()
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
-
- String fname = PARFOR_MR_RESULTMERGE_FNAME;
- fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
- fname = fname.replaceAll("%VAR%", String.valueOf(_resultVarsIDSeq.getNextID()));
-
- StringBuilder sb = new StringBuilder();
- sb.append(scratchSpaceLoc);
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(Lop.PROCESS_PREFIX);
- sb.append(DMLScript.getUUID());
- sb.append(fname);
-
- return sb.toString();
- }
-
- /**
- *
- * @return
- */
- private String constructDataPartitionsFileName()
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
-
- String fname = PARFOR_DATAPARTITIONS_FNAME;
- fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
- fname = fname.replaceAll("%VAR%", String.valueOf(_dpVarsIDSeq.getNextID()));
-
- StringBuilder sb = new StringBuilder();
- sb.append(scratchSpaceLoc);
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(Lop.PROCESS_PREFIX);
- sb.append(DMLScript.getUUID());
- sb.append(fname);
-
- return sb.toString();
- }
-
- /**
- *
- * @return
- */
- private long getMinMemory(ExecutionContext ec)
- {
- long ret = -1;
-
- //if forced remote exec and single node
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE
- && _execMode == PExecMode.REMOTE_MR
- && _optMode == POptMode.NONE )
- {
- try
- {
- ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
- OptTree tree = OptTreeConverter.createAbstractOptTree(-1, -1, sb, this, new HashSet<String>(), ec);
- CostEstimator est = new CostEstimatorHops( OptTreeConverter.getAbstractPlanMapping() );
- double mem = est.getEstimate(TestMeasure.MEMORY_USAGE, tree.getRoot());
-
- ret = (long) (mem * ( 1d/OptimizerUtils.MEM_UTIL_FACTOR ));
- }
- catch(Exception e)
- {
- LOG.error("Failed to analyze minmum memory requirements.", e);
- }
- }
-
- return ret;
- }
-
- private void setMemoryBudget()
- {
- if( _recompileMemoryBudget > 0 )
- {
- // store old budget for reset after exec
- _oldMemoryBudget = (double)InfrastructureAnalyzer.getLocalMaxMemory();
-
- // scale budget with applied mem util factor (inverted during getMemBudget() )
- long newMaxMem = (long) (_recompileMemoryBudget / OptimizerUtils.MEM_UTIL_FACTOR);
- InfrastructureAnalyzer.setLocalMaxMemory( newMaxMem );
- }
- }
-
- private void resetMemoryBudget()
- {
- if( _recompileMemoryBudget > 0 )
- {
- InfrastructureAnalyzer.setLocalMaxMemory((long)_oldMemoryBudget);
- }
- }
-
- private void resetOptimizerFlags()
- {
- //reset all state that was set but is not guaranteed to be overwritten by optimizer
- _variablesDPOriginal.removeAll();
- _iterablePredicateVarsOriginal = null;
- _colocatedDPMatrix = null;
- _replicationDP = WRITE_REPLICATION_FACTOR;
- _replicationExport = -1;
- _jvmReuse = true;
- _recompileMemoryBudget = -1;
- _enableRuntimePiggybacking = false;
- _variablesRP = null;
- _variablesECache = null;
- }
-
-
- /**
- * Helper class for parallel invocation of REMOTE_MR result merge for multiple variables.
- */
- private class ResultMergeWorker implements Runnable
- {
- private LocalTaskQueue<String> _q = null;
- private LocalVariableMap[] _refVars = null;
- private ExecutionContext _ec = null;
-
- public ResultMergeWorker( LocalTaskQueue<String> q, LocalVariableMap[] results, ExecutionContext ec )
- {
- _q = q;
- _refVars = results;
- _ec = ec;
- }
-
- @Override
- public void run()
- {
- try
- {
- while( true )
- {
- String varname = _q.dequeueTask();
- if( varname == LocalTaskQueue.NO_MORE_TASKS ) // task queue closed (no more tasks)
- break;
-
- MatrixObject out = null;
- synchronized( _ec.getVariables() ){
- out = (MatrixObject) _ec.getVariable(varname);
- }
-
- MatrixObject[] in = new MatrixObject[ _refVars.length ];
- for( int i=0; i< _refVars.length; i++ )
- in[i] = (MatrixObject) _refVars[i].get( varname );
- String fname = constructResultMergeFileName();
-
- ResultMerge rm = createResultMerge(_resultMerge, out, in, fname, _ec);
- MatrixObject outNew = null;
- if( USE_PARALLEL_RESULT_MERGE )
- outNew = rm.executeParallelMerge( _numThreads );
- else
- outNew = rm.executeSerialMerge();
-
- synchronized( _ec.getVariables() ){
- _ec.getVariables().put( varname, outNew);
- }
-
- //cleanup of intermediate result variables
- cleanWorkerResultVariables( _ec, out, in );
- }
- }
- catch(Exception ex)
- {
- LOG.error("Error executing result merge: ", ex);
- }
- }
- }
-
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in parfor program block generated from parfor statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/Program.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/Program.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/Program.java
deleted file mode 100644
index 08d7958..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/Program.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-
-
-public class Program
-{
-
- public static final String KEY_DELIM = "::";
-
- public ArrayList<ProgramBlock> _programBlocks;
-
- private HashMap<String, HashMap<String,FunctionProgramBlock>> _namespaceFunctions;
-
- public Program() throws DMLRuntimeException {
- _namespaceFunctions = new HashMap<String, HashMap<String,FunctionProgramBlock>>();
- _programBlocks = new ArrayList<ProgramBlock>();
- }
-
- /**
- *
- * @param namespace
- * @param fname
- * @param fpb
- */
- public synchronized void addFunctionProgramBlock(String namespace, String fname, FunctionProgramBlock fpb)
- {
- if (namespace == null)
- namespace = DMLProgram.DEFAULT_NAMESPACE;
-
- HashMap<String,FunctionProgramBlock> namespaceBlocks = null;
- namespaceBlocks = _namespaceFunctions.get(namespace);
- if (namespaceBlocks == null){
- namespaceBlocks = new HashMap<String,FunctionProgramBlock>();
- _namespaceFunctions.put(namespace,namespaceBlocks);
- }
-
- namespaceBlocks.put(fname,fpb);
- }
-
- /**
- *
- * @param namespace
- * @param fname
- */
- public synchronized void removeFunctionProgramBlock(String namespace, String fname)
- {
- if (namespace == null)
- namespace = DMLProgram.DEFAULT_NAMESPACE;
-
- HashMap<String,FunctionProgramBlock> namespaceBlocks = null;
- if( _namespaceFunctions.containsKey(namespace) ){
- namespaceBlocks = _namespaceFunctions.get(namespace);
- if( namespaceBlocks.containsKey(fname) )
- namespaceBlocks.remove(fname);
- }
- }
-
- /**
- *
- * @return
- */
- public synchronized HashMap<String,FunctionProgramBlock> getFunctionProgramBlocks(){
-
- HashMap<String,FunctionProgramBlock> retVal = new HashMap<String,FunctionProgramBlock>();
-
- //create copy of function program blocks
- for (String namespace : _namespaceFunctions.keySet()){
- HashMap<String,FunctionProgramBlock> namespaceFSB = _namespaceFunctions.get(namespace);
- for( Entry<String, FunctionProgramBlock> e: namespaceFSB.entrySet() ){
- String fname = e.getKey();
- FunctionProgramBlock fpb = e.getValue();
- String fKey = DMLProgram.constructFunctionKey(namespace, fname);
- retVal.put(fKey, fpb);
- }
- }
-
- return retVal;
- }
-
- /**
- *
- * @param namespace
- * @param fname
- * @return
- * @throws DMLRuntimeException
- */
- public synchronized FunctionProgramBlock getFunctionProgramBlock(String namespace, String fname) throws DMLRuntimeException{
-
- if (namespace == null) namespace = DMLProgram.DEFAULT_NAMESPACE;
-
- HashMap<String,FunctionProgramBlock> namespaceFunctBlocks = _namespaceFunctions.get(namespace);
- if (namespaceFunctBlocks == null)
- throw new DMLRuntimeException("namespace " + namespace + " is undefined");
- FunctionProgramBlock retVal = namespaceFunctBlocks.get(fname);
- if (retVal == null)
- throw new DMLRuntimeException("function " + fname + " is undefined in namespace " + namespace);
-
- return retVal;
- }
-
- public void addProgramBlock(ProgramBlock pb) {
- _programBlocks.add(pb);
- }
-
- public ArrayList<ProgramBlock> getProgramBlocks() {
- return _programBlocks;
- }
-
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- ec.initDebugProgramCounters();
-
- try
- {
- for (int i=0 ; i<_programBlocks.size() ; i++) {
- ec.updateDebugState(i);
- _programBlocks.get(i).execute(ec);
- }
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch(Exception e) {
- throw new DMLRuntimeException(e);
- }
-
- ec.clearDebugProgramCounters();
- }
-
-
- public void printMe() {
-
- for (ProgramBlock pb : this._programBlocks) {
- pb.printMe();
- }
- }
-}
[08/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/MatrixObject.java
deleted file mode 100644
index 839a830..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/MatrixObject.java
+++ /dev/null
@@ -1,1686 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.io.IOException;
-import java.lang.ref.SoftReference;
-
-import org.apache.commons.lang.mutable.MutableBoolean;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.spark.data.BroadcastObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDProperties;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.FileFormatProperties;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-
-
-/**
- * Represents a matrix in control program. This class contains method to read
- * matrices from HDFS and convert them to a specific format/representation. It
- * is also able to write several formats/representation of matrices to HDFS.
-
- * IMPORTANT: Preserve one-to-one correspondence between {@link MatrixObject}
- * and {@link MatrixBlock} objects, for cache purposes. Do not change a
- * {@link MatrixBlock} object without informing its {@link MatrixObject} object.
- *
- */
-public class MatrixObject extends CacheableData
-{
- private static final long serialVersionUID = 6374712373206495637L;
-
- /**
- * Current state of pinned variables, required for guarded collect.
- */
- private static ThreadLocal<Long> sizePinned = new ThreadLocal<Long>() {
- @Override protected Long initialValue() { return 0L; }
- };
-
- /**
- * Cache for actual data, evicted by garbage collector.
- */
- private SoftReference<MatrixBlock> _cache = null;
-
- /**
- * Container object that holds the actual data.
- */
- private MatrixBlock _data = null;
-
- /**
- * The name of HDFS file in which the data is backed up.
- */
- private String _hdfsFileName = null; // file name and path
-
- /**
- * Flag that indicates whether or not hdfs file exists.
- * It is used for improving the performance of "rmvar" instruction.
- * When it has value <code>false</code>, one can skip invocations to
- * utility functions such as MapReduceTool.deleteFileIfExistOnHDFS(),
- * which can be potentially expensive.
- */
- private boolean _hdfsFileExists = false;
-
- /**
- * <code>true</code> if the in-memory or evicted matrix may be different from
- * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
- * matrices should be the same.
- */
- private boolean _dirtyFlag = false;
-
- /**
- * Object that holds the metadata associated with the matrix, which
- * includes: 1) Matrix dimensions, if available 2) Number of non-zeros, if
- * available 3) Block dimensions, if applicable 4) InputInfo -- subsequent
- * operations that use this Matrix expect it to be in this format.
- *
- * When the matrix is written to HDFS (local file system, as well?), one
- * must get the OutputInfo that matches with InputInfo stored inside _mtd.
- */
- private MetaData _metaData = null;
-
- //additional names and flags
- private String _varName = ""; //plan variable name
- private String _cacheFileName = null; //local eviction file name
- private boolean _requiresLocalWrite = false; //flag if local write for read obj
- private boolean _isAcquireFromEmpty = false; //flag if read from status empty
- private boolean _cleanupFlag = true; //flag if obj unpinned (cleanup enabled)
- private boolean _updateInPlaceFlag = false; //flag if in-place update
-
- //spark-specific handles
- //note: we use the abstraction of LineageObjects for two reasons: (1) to keep track of cleanup
- //for lazily evaluated RDDs, and (2) as abstraction for environments that do not necessarily have spark libraries available
- private RDDObject _rddHandle = null; //RDD handle
- private BroadcastObject _bcHandle = null; //Broadcast handle
- private RDDProperties _rddProperties = null;
-
- /**
- * Information relevant to partitioned matrices.
- */
- private boolean _partitioned = false; //indicates if obj partitioned
- private PDataPartitionFormat _partitionFormat = null; //indicates how obj partitioned
- private int _partitionSize = -1; //indicates n for BLOCKWISE_N
- private String _partitionCacheName = null; //name of cache block
- private MatrixBlock _partitionInMemory = null;
-
- /**
- * Information relevant to specific external file formats
- */
- FileFormatProperties _formatProperties = null;
-
- public RDDProperties getRddProperties() {
- return _rddProperties;
- }
-
- public void setRddProperties(RDDProperties _rddProperties) {
- this._rddProperties = _rddProperties;
- }
-
- /**
- * Constructor that takes only the HDFS filename.
- */
- public MatrixObject (ValueType vt, String file)
- {
- this (vt, file, null); //HDFS file path
- }
-
- /**
- * Constructor that takes both HDFS filename and associated metadata.
- */
- public MatrixObject( ValueType vt, String file, MetaData mtd )
- {
- super (DataType.MATRIX, vt);
- _metaData = mtd;
- _hdfsFileName = file;
-
- _cache = null;
- _data = null;
- }
-
- /**
- * Copy constructor that copies meta data but NO data.
- *
- * @param mo
- */
- public MatrixObject( MatrixObject mo )
- {
- super(mo.getDataType(), mo.getValueType());
-
- _hdfsFileName = mo._hdfsFileName;
- _hdfsFileExists = mo._hdfsFileExists;
-
- MatrixFormatMetaData metaOld = (MatrixFormatMetaData)mo.getMetaData();
- _metaData = new MatrixFormatMetaData(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
- metaOld.getOutputInfo(), metaOld.getInputInfo());
-
- _varName = mo._varName;
- _cleanupFlag = mo._cleanupFlag;
- _updateInPlaceFlag = mo._updateInPlaceFlag;
- _partitioned = mo._partitioned;
- _partitionFormat = mo._partitionFormat;
- _partitionSize = mo._partitionSize;
- _partitionCacheName = mo._partitionCacheName;
- }
-
- public void setVarName(String s)
- {
- _varName = s;
- }
-
- public String getVarName()
- {
- return _varName;
- }
-
- @Override
- public void setMetaData(MetaData md)
- {
- _metaData = md;
- }
-
- @Override
- public MetaData getMetaData()
- {
- return _metaData;
- }
-
- @Override
- public void removeMetaData()
- {
- _metaData = null;
- }
-
- @Override
- public void updateMatrixCharacteristics (MatrixCharacteristics mc)
- {
- ((MatrixDimensionsMetaData)_metaData).setMatrixCharacteristics( mc );
- }
-
- /**
- * Make the matrix metadata consistent with the in-memory matrix data
- * @throws CacheException
- */
- public void refreshMetaData()
- throws CacheException
- {
- if ( _data == null || _metaData ==null ) //refresh only for existing data
- throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
- //we need to throw an exception, otherwise input/output format cannot be inferred
-
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
- mc.setDimension( _data.getNumRows(),
- _data.getNumColumns() );
- mc.setNonZeros( _data.getNonZeros() );
- }
-
- public void setFileFormatProperties(FileFormatProperties formatProperties) {
- _formatProperties = formatProperties;
- }
-
- public FileFormatProperties getFileFormatProperties() {
- return _formatProperties;
- }
-
- public boolean isFileExists()
- {
- return _hdfsFileExists;
- }
-
- public void setFileExists( boolean flag )
- {
- _hdfsFileExists = flag;
- }
-
- public String getFileName()
- {
- return _hdfsFileName;
- }
-
- public synchronized void setFileName( String file )
- {
- if (!_hdfsFileName.equals (file))
- {
- _hdfsFileName = file;
- if( ! isEmpty(true) )
- _dirtyFlag = true;
- }
- }
-
- /**
- *
- * @return
- */
- public long getNumRows ()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- return mc.getRows ();
- }
-
- /**
- *
- * @return
- */
- public long getNumColumns()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- return mc.getCols ();
- }
-
- /**
- *
- * @return
- */
- public long getNumRowsPerBlock()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- return mc.getRowsPerBlock();
- }
-
- /**
- *
- * @return
- */
- public long getNumColumnsPerBlock()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- return mc.getColsPerBlock();
- }
-
- /**
- *
- * @return
- */
- public long getNnz()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- return mc.getNonZeros();
- }
-
- /**
- *
- * @return
- */
- public double getSparsity()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
-
- return ((double)mc.getNonZeros())/mc.getRows()/mc.getCols();
- }
-
- /**
- *
- * @return
- */
- public MatrixCharacteristics getMatrixCharacteristics()
- {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- return meta.getMatrixCharacteristics();
- }
-
- /**
- * <code>true</code> if the in-memory or evicted matrix may be different from
- * the matrix located at {@link #_hdfsFileName}; <code>false</code> if the two
- * matrices are supposed to be the same.
- */
- public boolean isDirty ()
- {
- return _dirtyFlag;
- }
-
- public String toString()
- {
- StringBuilder str = new StringBuilder();
- str.append("Matrix: ");
- str.append(_hdfsFileName + ", ");
- //System.out.println(_hdfsFileName);
- if ( _metaData instanceof NumItemsByEachReducerMetaData ) {
- str.append("NumItemsByEachReducerMetaData");
- }
- else
- {
- try
- {
- MatrixFormatMetaData md = (MatrixFormatMetaData)_metaData;
- if ( md != null ) {
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)_metaData).getMatrixCharacteristics();
- str.append(mc.toString());
-
- InputInfo ii = md.getInputInfo();
- if ( ii == null )
- str.append("null");
- else {
- str.append(", ");
- str.append(InputInfo.inputInfoToString(ii));
- }
- }
- else {
- str.append("null, null");
- }
- }
- catch(Exception ex)
- {
- LOG.error(ex);
- }
- }
- str.append(", ");
- str.append(isDirty() ? "dirty" : "not-dirty");
-
- return str.toString();
- }
-
- public RDDObject getRDDHandle()
- {
- return _rddHandle;
- }
-
- public void setRDDHandle( RDDObject rdd )
- {
- //cleanup potential old back reference
- if( _rddHandle != null )
- _rddHandle.setBackReference(null);
-
- //add new rdd handle
- _rddHandle = rdd;
- if( _rddHandle != null )
- rdd.setBackReference(this);
- }
-
- public BroadcastObject getBroadcastHandle()
- {
- return _bcHandle;
- }
-
- public void setBroadcastHandle( BroadcastObject bc )
- {
- //cleanup potential old back reference
- if( _bcHandle != null )
- _bcHandle.setBackReference(null);
-
- //add new broadcast handle
- _bcHandle = bc;
- if( _bcHandle != null )
- bc.setBackReference(this);
- }
-
-
- // *********************************************
- // *** ***
- // *** HIGH-LEVEL METHODS THAT SPECIFY ***
- // *** THE LOCKING AND CACHING INTERFACE ***
- // *** ***
- // *********************************************
-
-
- /**
- * Acquires a shared "read-only" lock, produces the reference to the matrix data,
- * restores the matrix to main memory, reads from HDFS if needed.
- *
- * Synchronized because there might be parallel threads (parfor local) that
- * access the same MatrixObjectNew object (in case it was created before the loop).
- *
- * In-Status: EMPTY, EVICTABLE, EVICTED, READ;
- * Out-Status: READ(+1).
- *
- * @return the matrix data reference
- * @throws CacheException
- */
- public synchronized MatrixBlock acquireRead()
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Acquire read "+_varName);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- if ( !isAvailableToRead() )
- throw new CacheStatusException ("MatrixObject not available to read.");
-
- //get object from cache
- if( _data == null )
- getCache();
-
- //read data from HDFS/RDD if required
- //(probe data for cache_nowrite / jvm_reuse)
- if( isEmpty(true) && _data==null )
- {
- try
- {
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementHDFSHits();
-
- if( getRDDHandle()==null || getRDDHandle().allowsShortCircuitRead() )
- {
- //check filename
- if( _hdfsFileName == null )
- throw new CacheException("Cannot read matrix for empty filename.");
-
- //read matrix from hdfs
- _data = readMatrixFromHDFS( _hdfsFileName );
-
- //mark for initial local write despite read operation
- _requiresLocalWrite = CACHING_WRITE_CACHE_ON_READ;
- }
- else
- {
- //read matrix from rdd (incl execute pending rdd operations)
- MutableBoolean writeStatus = new MutableBoolean();
- _data = readMatrixFromRDD( getRDDHandle(), writeStatus );
-
- //mark for initial local write (prevent repeated execution of rdd operations)
- if( writeStatus.booleanValue() )
- _requiresLocalWrite = CACHING_WRITE_CACHE_ON_READ;
- else
- _requiresLocalWrite = true;
- }
-
- _dirtyFlag = false;
- }
- catch (IOException e)
- {
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
- }
-
- _isAcquireFromEmpty = true;
- }
- else if( DMLScript.STATISTICS )
- {
- if( _data!=null )
- CacheStatistics.incrementMemHits();
- }
-
- //cache status maintenance
- super.acquire( false, _data==null );
- updateStatusPinned(true);
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementAcquireRTime(t1-t0);
- }
-
- return _data;
- }
-
- /**
- * Acquires the exclusive "write" lock for a thread that wants to change matrix
- * cell values. Produces the reference to the matrix data, restores the matrix
- * to main memory, reads from HDFS if needed.
- *
- * In-Status: EMPTY, EVICTABLE, EVICTED;
- * Out-Status: MODIFY.
- *
- * @return the matrix data reference
- * @throws CacheException
- */
- public synchronized MatrixBlock acquireModify()
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Acquire modify "+_varName);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- if ( !isAvailableToModify() )
- throw new CacheStatusException("MatrixObject not available to modify.");
-
- //get object from cache
- if( _data == null )
- getCache();
-
- //read data from HDFS if required
- if( isEmpty(true) && _data == null )
- {
- //check filename
- if( _hdfsFileName == null )
- throw new CacheException("Cannot read matrix for empty filename.");
-
- //load data
- try
- {
- _data = readMatrixFromHDFS( _hdfsFileName );
- }
- catch (IOException e)
- {
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
- }
- }
-
- //cache status maintenance
- super.acquire( true, _data==null );
- updateStatusPinned(true);
- _dirtyFlag = true;
- _isAcquireFromEmpty = false;
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementAcquireMTime(t1-t0);
- }
-
- return _data;
- }
-
- /**
- * Acquires the exclusive "write" lock for a thread that wants to throw away the
- * old matrix data and link up with new matrix data. Abandons the old matrix data
- * without reading it. Sets the new matrix data reference.
-
- * In-Status: EMPTY, EVICTABLE, EVICTED;
- * Out-Status: MODIFY.
- *
- * @param newData : the new matrix data reference
- * @return the matrix data reference, which is the same as the argument
- * @throws CacheException
- */
- public synchronized MatrixBlock acquireModify(MatrixBlock newData)
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Acquire modify newdata "+_varName);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- if (! isAvailableToModify ())
- throw new CacheStatusException ("MatrixObject not available to modify.");
-
- //clear old data
- clearData();
-
- //cache status maintenance
- super.acquire (true, false); //no need to load evicted matrix
- _dirtyFlag = true;
- _isAcquireFromEmpty = false;
-
- //set references to new data
- if (newData == null)
- throw new CacheException("acquireModify with empty matrix block.");
- _data = newData;
- updateStatusPinned(true);
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementAcquireMTime(t1-t0);
- }
-
- return _data;
- }
-
- /**
- * Releases the shared ("read-only") or exclusive ("write") lock. Updates
- * the matrix size, last-access time, metadata, etc.
- *
- * Synchronized because there might be parallel threads (parfor local) that
- * access the same MatrixObjectNew object (in case it was created before the loop).
- *
- * In-Status: READ, MODIFY;
- * Out-Status: READ(-1), EVICTABLE, EMPTY.
- *
- * @throws CacheStatusException
- */
- public synchronized void release()
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Release "+_varName);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- boolean write = false;
- if ( isModify() )
- {
- //set flags for write
- write = true;
- _dirtyFlag = true;
-
- //update meta data
- refreshMetaData();
- }
-
- //compact empty in-memory block
- if( _data.isEmptyBlock(false) && _data.isAllocated() )
- _data.cleanupBlock(true, true);
-
- //cache status maintenance (pass cacheNoWrite flag)
- super.release(_isAcquireFromEmpty && !_requiresLocalWrite);
- updateStatusPinned(false);
-
- if( isCachingActive() //only if caching is enabled (otherwise keep everything in mem)
- && isCached(true) //not empty and not read/modify
- && !isUpdateInPlace() //pinned result variable
- && !isBelowCachingThreshold() ) //min size for caching
- {
- if( write || _requiresLocalWrite )
- {
- //evict blob
- String filePath = getCacheFilePathAndName();
- try {
- writeMatrix (filePath);
- }
- catch (Exception e)
- {
- throw new CacheException("Eviction to local path " + filePath + " ("+_varName+") failed.", e);
- }
- _requiresLocalWrite = false;
- }
-
- //create cache
- createCache();
- _data = null;
- }
- else if( LOG.isTraceEnabled() ){
- LOG.trace("Var "+_varName+" not subject to caching: rows="+_data.getNumRows()+", cols="+_data.getNumColumns()+", state="+getStatusAsString());
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementReleaseTime(t1-t0);
- }
- }
-
- /**
- * Sets the matrix data reference to <code>null</code>, abandons the old matrix.
- * Makes the "envelope" empty. Run it to finalize the matrix (otherwise the
- * evicted matrix file may remain undeleted).
- *
- * In-Status: EMPTY, EVICTABLE, EVICTED;
- * Out-Status: EMPTY.
- * @throws CacheException
- */
- public synchronized void clearData()
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Clear data "+_varName);
-
- // check if cleanup enabled and possible
- if( !_cleanupFlag )
- return; // do nothing
- if( !isAvailableToModify() )
- throw new CacheStatusException ("MatrixObject (" + this.getDebugName() + ") not available to modify. Status = " + this.getStatusAsString() + ".");
-
- // clear existing WB / FS representation (but prevent unnecessary probes)
- if( !(isEmpty(true)||(_data!=null && isBelowCachingThreshold())
- ||(_data!=null && !isCachingActive()) )) //additional condition for JMLC
- freeEvictedBlob();
-
- // clear the in-memory data
- _data = null;
- clearCache();
-
- // clear rdd/broadcast back refs
- if( _rddHandle != null )
- _rddHandle.setBackReference(null);
- if( _bcHandle != null )
- _bcHandle.setBackReference(null);
-
- // change object state EMPTY
- _dirtyFlag = false;
- setEmpty();
- }
-
- public synchronized void exportData()
- throws CacheException
- {
- exportData( -1 );
- }
-
- /**
- * Writes, or flushes, the matrix data to HDFS.
- *
- * In-Status: EMPTY, EVICTABLE, EVICTED, READ;
- * Out-Status: EMPTY, EVICTABLE, EVICTED, READ.
- *
- * @throws CacheException
- */
- public synchronized void exportData( int replication )
- throws CacheException
- {
- exportData(_hdfsFileName, null, replication, null);
- _hdfsFileExists = true;
- }
-
- /**
- *
- * @param fName
- * @param outputFormat
- * @param formatProperties
- * @throws CacheException
- */
- public synchronized void exportData (String fName, String outputFormat, FileFormatProperties formatProperties)
- throws CacheException
- {
- exportData(fName, outputFormat, -1, formatProperties);
- }
-
- /**
- *
- * @param fName
- * @param outputFormat
- * @throws CacheException
- */
- public synchronized void exportData (String fName, String outputFormat)
- throws CacheException
- {
- exportData(fName, outputFormat, -1, null);
- }
-
- /**
- * Synchronized because there might be parallel threads (parfor local) that
- * access the same MatrixObjectNew object (in case it was created before the loop).
- * If all threads export the same data object concurrently it results in errors
- * because they all write to the same file. Efficiency for loops and parallel threads
- * is achieved by checking if the in-memory matrix block is dirty.
- *
- * NOTE: MB: we do not use dfs copy from local (evicted) to HDFS because this would ignore
- * the output format and most importantly would bypass reblocking during write (which effects the
- * potential degree of parallelism). However, we copy files on HDFS if certain criteria are given.
- *
- * @param fName
- * @param outputFormat
- * @throws CacheException
- */
- public synchronized void exportData (String fName, String outputFormat, int replication, FileFormatProperties formatProperties)
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Export data "+_varName+" "+fName);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- //prevent concurrent modifications
- if ( !isAvailableToRead() )
- throw new CacheStatusException ("MatrixObject not available to read.");
-
- LOG.trace("Exporting " + this.getDebugName() + " to " + fName + " in format " + outputFormat);
-
- boolean pWrite = false; // !fName.equals(_hdfsFileName); //persistent write flag
- if ( fName.equals(_hdfsFileName) ) {
- _hdfsFileExists = true;
- pWrite = false;
- }
- else {
- pWrite = true; // i.e., export is called from "write" instruction
- }
-
- //actual export (note: no direct transfer of local copy in order to ensure blocking (and hence, parallelism))
- if( isDirty() || //use dirty for skipping parallel exports
- (pWrite && !isEqualOutputFormat(outputFormat)) )
- {
- // CASE 1: dirty in-mem matrix or pWrite w/ different format (write matrix to fname; load into memory if evicted)
- // a) get the matrix
- if( isEmpty(true) )
- {
- //read data from HDFS if required (never read before), this applies only to pWrite w/ different output formats
- //note: for large rdd outputs, we compile dedicated writespinstructions (no need to handle this here)
- try
- {
- if( getRDDHandle()==null || getRDDHandle().allowsShortCircuitRead() )
- _data = readMatrixFromHDFS( _hdfsFileName );
- else
- _data = readMatrixFromRDD( getRDDHandle(), new MutableBoolean() );
- _dirtyFlag = false;
- }
- catch (IOException e)
- {
- throw new CacheIOException("Reading of " + _hdfsFileName + " ("+_varName+") failed.", e);
- }
- }
- //get object from cache
- if( _data == null )
- getCache();
- super.acquire( false, _data==null ); //incl. read matrix if evicted
-
- // b) write the matrix
- try
- {
- writeMetaData( fName, outputFormat, formatProperties );
- writeMatrixToHDFS( fName, outputFormat, replication, formatProperties );
- if ( !pWrite )
- _dirtyFlag = false;
- }
- catch (Exception e)
- {
- throw new CacheIOException ("Export to " + fName + " failed.", e);
- }
- finally
- {
- release();
- }
- }
- else if( pWrite ) // pwrite with same output format
- {
- //CASE 2: matrix already in same format but different file on hdfs (copy matrix to fname)
- try
- {
- MapReduceTool.deleteFileIfExistOnHDFS(fName);
- MapReduceTool.deleteFileIfExistOnHDFS(fName+".mtd");
- if( getRDDHandle()==null || getRDDHandle().allowsShortCircuitRead() )
- MapReduceTool.copyFileOnHDFS( _hdfsFileName, fName );
- else //write might trigger rdd operations and nnz maintenance
- writeMatrixFromRDDtoHDFS(getRDDHandle(), fName, outputFormat);
- writeMetaData( fName, outputFormat, formatProperties );
- }
- catch (Exception e) {
- throw new CacheIOException ("Export to " + fName + " failed.", e);
- }
- }
- else if( getRDDHandle()!=null && //pending rdd operation
- !getRDDHandle().allowsShortCircuitRead() )
- {
- //CASE 3: pending rdd operation (other than checkpoints)
- try
- {
- writeMatrixFromRDDtoHDFS(getRDDHandle(), fName, outputFormat);
- writeMetaData( fName, outputFormat, formatProperties );
- }
- catch (Exception e) {
- throw new CacheIOException ("Export to " + fName + " failed.", e);
- }
- }
- else
- {
- //CASE 4: data already in hdfs (do nothing, no need for export)
- LOG.trace(this.getDebugName() + ": Skip export to hdfs since data already exists.");
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementExportTime(t1-t0);
- }
- }
-
- /**
- *
- * @param fName
- * @param outputFormat
- * @return
- * @throws CacheIOException
- */
- public synchronized boolean moveData(String fName, String outputFormat)
- throws CacheIOException
- {
- boolean ret = false;
-
- try
- {
- //ensure input file is persistent on hdfs (pending RDD operations),
- //file might have been written during export or collect via write/read
- if( getRDDHandle() != null && !MapReduceTool.existsFileOnHDFS(_hdfsFileName) ) {
- writeMatrixFromRDDtoHDFS(getRDDHandle(), _hdfsFileName, outputFormat);
- }
-
- //export or rename to target file on hdfs
- if( isDirty() || (!isEqualOutputFormat(outputFormat) && isEmpty(true)))
- {
- exportData(fName, outputFormat);
- ret = true;
- }
- else if( isEqualOutputFormat(outputFormat) )
- {
- MapReduceTool.deleteFileIfExistOnHDFS(fName);
- MapReduceTool.deleteFileIfExistOnHDFS(fName+".mtd");
- MapReduceTool.renameFileOnHDFS( _hdfsFileName, fName );
- writeMetaData( fName, outputFormat, null );
- ret = true;
- }
- }
- catch (Exception e)
- {
- throw new CacheIOException ("Move to " + fName + " failed.", e);
- }
-
- return ret;
- }
-
-
- // *********************************************
- // *** ***
- // *** HIGH-LEVEL PUBLIC METHODS ***
- // *** FOR PARTITIONED MATRIX ACCESS ***
- // *** (all other methods still usable) ***
- // *** ***
- // *********************************************
-
- /**
- * @param n
- *
- */
- public void setPartitioned( PDataPartitionFormat format, int n )
- {
- _partitioned = true;
- _partitionFormat = format;
- _partitionSize = n;
- }
-
-
- public void unsetPartitioned()
- {
- _partitioned = false;
- _partitionFormat = null;
- _partitionSize = -1;
- }
-
- /**
- *
- * @return
- */
- public boolean isPartitioned()
- {
- return _partitioned;
- }
-
- public PDataPartitionFormat getPartitionFormat()
- {
- return _partitionFormat;
- }
-
- public int getPartitionSize()
- {
- return _partitionSize;
- }
-
- public synchronized void setInMemoryPartition(MatrixBlock block)
- {
- _partitionInMemory = block;
- }
-
- /**
- * NOTE: for reading matrix partitions, we could cache (in its real sense) the read block
- * with soft references (no need for eviction, as partitioning only applied for read-only matrices).
- * However, since we currently only support row- and column-wise partitioning caching is not applied yet.
- * This could be changed once we also support column-block-wise and row-block-wise. Furthermore,
- * as we reject to partition vectors and support only full row or column indexing, no metadata (apart from
- * the partition flag) is required.
- *
- * @param pred
- * @return
- * @throws CacheException
- */
- public synchronized MatrixBlock readMatrixPartition( IndexRange pred )
- throws CacheException
- {
- if( LOG.isTraceEnabled() )
- LOG.trace("Acquire partition "+_varName+" "+pred);
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- if ( !_partitioned )
- throw new CacheStatusException ("MatrixObject not available to indexed read.");
-
- //return static partition of set from outside of the program
- if( _partitionInMemory != null )
- return _partitionInMemory;
-
- MatrixBlock mb = null;
-
- try
- {
- boolean blockwise = (_partitionFormat==PDataPartitionFormat.ROW_BLOCK_WISE || _partitionFormat==PDataPartitionFormat.COLUMN_BLOCK_WISE);
-
- //preparations for block wise access
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
-
- //get filename depending on format
- String fname = getPartitionFileName( pred, brlen, bclen );
-
- //probe cache
- if( blockwise && _partitionCacheName != null && _partitionCacheName.equals(fname) )
- {
- mb = _cache.get(); //try getting block from cache
- }
-
- if( mb == null ) //block not in cache
- {
- //get rows and cols
- long rows = -1;
- long cols = -1;
- switch( _partitionFormat )
- {
- case ROW_WISE:
- rows = 1;
- cols = mc.getCols();
- break;
- case ROW_BLOCK_WISE:
- rows = brlen;
- cols = mc.getCols();
- break;
- case COLUMN_WISE:
- rows = mc.getRows();
- cols = 1;
- break;
- case COLUMN_BLOCK_WISE:
- rows = mc.getRows();
- cols = bclen;
- break;
- default:
- throw new CacheException("Unsupported partition format: "+_partitionFormat);
- }
-
-
- //read the
- if( MapReduceTool.existsFileOnHDFS(fname) )
- mb = readMatrixFromHDFS( fname, rows, cols );
- else
- {
- mb = new MatrixBlock((int)rows, (int)cols, true);
- LOG.warn("Reading empty matrix partition "+fname);
- }
- }
-
- //post processing
- if( blockwise )
- {
- //put block into cache
- _partitionCacheName = fname;
- _cache = new SoftReference<MatrixBlock>(mb);
-
- if( _partitionFormat == PDataPartitionFormat.ROW_BLOCK_WISE )
- {
- int rix = (int)((pred.rowStart-1)%brlen);
- mb = mb.sliceOperations(rix, rix, (int)(pred.colStart-1), (int)(pred.colEnd-1), new MatrixBlock());
- }
- if( _partitionFormat == PDataPartitionFormat.COLUMN_BLOCK_WISE )
- {
- int cix = (int)((pred.colStart-1)%bclen);
- mb = mb.sliceOperations((int)(pred.rowStart-1), (int)(pred.rowEnd-1), cix, cix, new MatrixBlock());
- }
- }
-
- //NOTE: currently no special treatment of non-existing partitions necessary
- // because empty blocks are written anyway
- }
- catch(Exception ex)
- {
- throw new CacheException(ex);
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- CacheStatistics.incrementAcquireRTime(t1-t0);
- }
-
- return mb;
- }
-
-
- /**
- *
- * @param pred
- * @return
- * @throws CacheStatusException
- */
- public String getPartitionFileName( IndexRange pred, int brlen, int bclen )
- throws CacheStatusException
- {
- if ( !_partitioned )
- throw new CacheStatusException ("MatrixObject not available to indexed read.");
-
- StringBuilder sb = new StringBuilder();
- sb.append(_hdfsFileName);
-
- switch( _partitionFormat )
- {
- case ROW_WISE:
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(pred.rowStart);
- break;
- case ROW_BLOCK_WISE:
- sb.append(Lop.FILE_SEPARATOR);
- sb.append((pred.rowStart-1)/brlen+1);
- break;
- case COLUMN_WISE:
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(pred.colStart);
- break;
- case COLUMN_BLOCK_WISE:
- sb.append(Lop.FILE_SEPARATOR);
- sb.append((pred.colStart-1)/bclen+1);
- break;
- default:
- throw new CacheStatusException ("MatrixObject not available to indexed read.");
- }
-
- return sb.toString();
- }
-
-
-
- // *********************************************
- // *** ***
- // *** LOW-LEVEL PROTECTED METHODS ***
- // *** EXTEND CACHEABLE DATA ***
- // *** ONLY CALLED BY THE SUPERCLASS ***
- // *** ***
- // *********************************************
-
-
- @Override
- protected boolean isBlobPresent()
- {
- return (_data != null);
- }
-
- @Override
- protected void evictBlobFromMemory ( MatrixBlock mb )
- throws CacheIOException
- {
- throw new CacheIOException("Redundant explicit eviction.");
- }
-
- @Override
- protected void restoreBlobIntoMemory ()
- throws CacheIOException
- {
- long begin = 0;
-
- if( LOG.isTraceEnabled() ) {
- LOG.trace("RESTORE of Matrix "+_varName+", "+_hdfsFileName);
- begin = System.currentTimeMillis();
- }
-
- String filePath = getCacheFilePathAndName();
-
- if( LOG.isTraceEnabled() )
- LOG.trace ("CACHE: Restoring matrix... " + _varName + " HDFS path: " +
- (_hdfsFileName == null ? "null" : _hdfsFileName) + ", Restore from path: " + filePath);
-
- if (_data != null)
- throw new CacheIOException (filePath + " : Cannot restore on top of existing in-memory data.");
-
- try
- {
- _data = readMatrix(filePath);
- }
- catch (IOException e)
- {
- throw new CacheIOException (filePath + " : Restore failed.", e);
- }
-
- //check for success
- if (_data == null)
- throw new CacheIOException (filePath + " : Restore failed.");
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Restoring matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
- }
-
- @Override
- protected void freeEvictedBlob()
- {
- String cacheFilePathAndName = getCacheFilePathAndName();
- long begin = 0;
- if( LOG.isTraceEnabled() ){
- LOG.trace("CACHE: Freeing evicted matrix... " + _varName + " HDFS path: " +
- (_hdfsFileName == null ? "null" : _hdfsFileName) + " Eviction path: " + cacheFilePathAndName);
- begin = System.currentTimeMillis();
- }
-
- LazyWriteBuffer.deleteMatrix(cacheFilePathAndName);
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Freeing evicted matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
- }
-
- @Override
- protected boolean isBelowCachingThreshold()
- {
- long rlen = _data.getNumRows();
- long clen = _data.getNumColumns();
- long nnz = _data.getNonZeros();
-
- //get in-memory size (assume dense, if nnz unknown)
- double sparsity = OptimizerUtils.getSparsity( rlen, clen, nnz );
- double size = MatrixBlock.estimateSizeInMemory( rlen, clen, sparsity );
-
- return ( !_data.isAllocated() || size <= CACHING_THRESHOLD );
- }
-
- // *******************************************
- // *** ***
- // *** LOW-LEVEL PRIVATE METHODS ***
- // *** FOR MATRIX I/O ***
- // *** ***
- // *******************************************
-
- private boolean isUpdateInPlace()
- {
- return _updateInPlaceFlag;
- }
-
- /**
- *
- */
- private String getCacheFilePathAndName ()
- {
- if( _cacheFileName==null )
- {
- StringBuilder sb = new StringBuilder();
- sb.append(CacheableData.cacheEvictionLocalFilePath);
- sb.append(CacheableData.cacheEvictionLocalFilePrefix);
- sb.append(String.format ("%09d", getUniqueCacheID()));
- sb.append(CacheableData.cacheEvictionLocalFileExtension);
-
- _cacheFileName = sb.toString();
- }
-
- return _cacheFileName;
- }
-
- /**
- *
- * @param filePathAndName
- * @return
- * @throws IOException
- */
- private MatrixBlock readMatrix (String filePathAndName)
- throws IOException
- {
- return LazyWriteBuffer.readMatrix(filePathAndName);
- }
-
- /**
- *
- * @param filePathAndName
- * @return
- * @throws IOException
- */
- private MatrixBlock readMatrixFromHDFS(String filePathAndName)
- throws IOException
- {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
- return readMatrixFromHDFS( filePathAndName, mc.getRows(), mc.getCols() );
- }
-
- /**
- *
- * @param rdd
- * @return
- * @throws IOException
- */
- private MatrixBlock readMatrixFromRDD(RDDObject rdd, MutableBoolean writeStatus)
- throws IOException
- {
- //note: the read of a matrix block from an RDD might trigger
- //lazy evaluation of pending transformations.
- RDDObject lrdd = rdd;
-
- //prepare return status (by default only collect)
- writeStatus.setValue(false);
-
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
- MatrixBlock mb = null;
- try
- {
- //prevent unnecessary collect through rdd checkpoint
- if( rdd.allowsShortCircuitCollect() ) {
- lrdd = (RDDObject)rdd.getLineageChilds().get(0);
- }
-
- //obtain matrix block from RDD
- int rlen = (int)mc.getRows();
- int clen = (int)mc.getCols();
- int brlen = (int)mc.getRowsPerBlock();
- int bclen = (int)mc.getColsPerBlock();
- long nnz = mc.getNonZeros();
-
- //guarded rdd collect
- if( !OptimizerUtils.checkSparkCollectMemoryBudget(rlen, clen, brlen, bclen, nnz, sizePinned.get()) ) {
- //write RDD to hdfs and read to prevent invalid collect mem consumption
- //note: lazy, partition-at-a-time collect (toLocalIterator) was significantly slower
- if( !MapReduceTool.existsFileOnHDFS(_hdfsFileName) ) { //prevent overwrite existing file
- long newnnz = SparkExecutionContext.writeRDDtoHDFS(lrdd, _hdfsFileName, iimd.getOutputInfo());
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
- ((RDDObject)rdd).setHDFSFile(true); //mark rdd as hdfs file (for restore)
- writeStatus.setValue(true); //mark for no cache-write on read
- }
- mb = readMatrixFromHDFS(_hdfsFileName);
- }
- else {
- //collect matrix block from RDD
- mb = SparkExecutionContext.toMatrixBlock(lrdd, rlen, clen, brlen, bclen, nnz);
- }
- }
- catch(DMLRuntimeException ex) {
- throw new IOException(ex);
- }
-
- //sanity check correct output
- if( mb == null ) {
- throw new IOException("Unable to load matrix from rdd: "+lrdd.getVarName());
- }
-
- return mb;
- }
-
- /**
- *
- * @param rdd
- * @param fname
- * @param outputFormat
- * @throws DMLRuntimeException
- */
- private void writeMatrixFromRDDtoHDFS(RDDObject rdd, String fname, String outputFormat)
- throws DMLRuntimeException
- {
- //prepare output info
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat)
- : InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
-
- //note: the write of an RDD to HDFS might trigger
- //lazy evaluation of pending transformations.
- long newnnz = SparkExecutionContext.writeRDDtoHDFS(rdd, fname, oinfo);
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
- }
-
- /**
- *
- * @param filePathAndName
- * @param rlen
- * @param clen
- * @return
- * @throws IOException
- */
- private MatrixBlock readMatrixFromHDFS(String filePathAndName, long rlen, long clen)
- throws IOException
- {
- long begin = 0;
-
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
-
- if( LOG.isTraceEnabled() ){
- LOG.trace("Reading matrix from HDFS... " + _varName + " Path: " + filePathAndName
- + ", dimensions: [" + mc.getRows() + ", " + mc.getCols() + ", " + mc.getNonZeros() + "]");
- begin = System.currentTimeMillis();
- }
-
- double sparsity = ( mc.getNonZeros() >= 0 ? ((double)mc.getNonZeros())/(mc.getRows()*mc.getCols()) : 1.0d) ; //expected sparsity
- MatrixBlock newData = DataConverter.readMatrixFromHDFS(filePathAndName, iimd.getInputInfo(),
- rlen, clen, mc.getRowsPerBlock(), mc.getColsPerBlock(), sparsity, _formatProperties);
-
- //sanity check correct output
- if( newData == null ) {
- throw new IOException("Unable to load matrix from file: "+filePathAndName);
- }
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Reading Completed: " + (System.currentTimeMillis()-begin) + " msec.");
-
- return newData;
- }
-
- /**
- *
- * @param filePathAndName
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void writeMatrix (String filePathAndName)
- throws DMLRuntimeException, IOException
- {
- LazyWriteBuffer.writeMatrix(filePathAndName, _data);
- }
-
- /**
- * Writes in-memory matrix to HDFS in a specified format.
- *
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void writeMatrixToHDFS (String filePathAndName, String outputFormat, int replication, FileFormatProperties formatProperties)
- throws DMLRuntimeException, IOException
- {
- long begin = 0;
- if( LOG.isTraceEnabled() ){
- LOG.trace (" Writing matrix to HDFS... " + _varName + " Path: " + filePathAndName + ", Format: " +
- (outputFormat != null ? outputFormat : "inferred from metadata"));
- begin = System.currentTimeMillis();
- }
-
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
-
- if (_data != null)
- {
- // Get the dimension information from the metadata stored within MatrixObject
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics ();
- // Write the matrix to HDFS in requested format
- OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat)
- : InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
-
- // when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
- // note: this is only required if singlenode (due to binarycell default)
- if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
- {
- DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros()), replication, formatProperties);
- }
- else {
- DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, mc, replication, formatProperties);
- }
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Writing matrix to HDFS ("+filePathAndName+") - COMPLETED... " + (System.currentTimeMillis()-begin) + " msec.");
- }
- else if( LOG.isTraceEnabled() )
- {
- LOG.trace ("Writing matrix to HDFS ("+filePathAndName+") - NOTHING TO WRITE (_data == null).");
- }
-
- if( DMLScript.STATISTICS )
- CacheStatistics.incrementHDFSWrites();
- }
-
- /**
- *
- * @param filePathAndName
- * @param outputFormat
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void writeMetaData (String filePathAndName, String outputFormat, FileFormatProperties formatProperties)
- throws DMLRuntimeException, IOException
- {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
-
- if (iimd != null)
- {
- // Write the matrix to HDFS in requested format
- OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat)
- : InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
-
- if ( oinfo != OutputInfo.MatrixMarketOutputInfo ) {
- // Get the dimension information from the metadata stored within MatrixObject
- MatrixCharacteristics mc = iimd.getMatrixCharacteristics ();
-
- // when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
- // note: this is only required if singlenode (due to binarycell default)
- if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
- {
- mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros());
- }
- MapReduceTool.writeMetaDataFile (filePathAndName + ".mtd", valueType, mc, oinfo, formatProperties);
- }
- }
- else {
- throw new DMLRuntimeException("Unexpected error while writing mtd file (" + filePathAndName + ") -- metadata is null.");
- }
- }
-
- /**
- *
- * @param outputFormat
- * @return
- */
- private boolean isEqualOutputFormat( String outputFormat )
- {
- boolean ret = true;
-
- if( outputFormat != null )
- {
- try
- {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- OutputInfo oi1 = InputInfo.getMatchingOutputInfo( iimd.getInputInfo() );
- OutputInfo oi2 = OutputInfo.stringToOutputInfo( outputFormat );
- if( oi1 != oi2 )
- {
- ret = false;
- }
- }
- catch(Exception ex)
- {
- ret = false;
- }
- }
-
- return ret;
- }
-
- @Override
- public synchronized String getDebugName()
- {
- int maxLength = 23;
- String debugNameEnding = (_hdfsFileName == null ? "null" :
- (_hdfsFileName.length() < maxLength ? _hdfsFileName : "..." +
- _hdfsFileName.substring (_hdfsFileName.length() - maxLength + 3)));
- return _varName + " " + debugNameEnding;
- }
-
-
- // *******************************************
- // *** ***
- // *** LOW-LEVEL PRIVATE METHODS ***
- // *** FOR SOFTREFERENCE CACHE ***
- // *** ***
- // *******************************************
-
- /**
- *
- */
- private void createCache( )
- {
- _cache = new SoftReference<MatrixBlock>( _data );
- }
-
- /**
- *
- */
- private void getCache()
- {
- if( _cache !=null )
- {
- _data = _cache.get();
- clearCache();
- }
- }
-
- /**
- *
- */
- private void clearCache()
- {
- if( _cache != null )
- {
- _cache.clear();
- _cache = null;
- }
- }
-
- /**
- *
- * @param add
- */
- private void updateStatusPinned(boolean add) {
- if( _data != null ) { //data should never be null
- long size = sizePinned.get();
- size += (add ? 1 : -1) * _data.getSizeInMemory();
- sizePinned.set( Math.max(size,0) );
- }
- }
-
- /**
- * see clear data
- *
- * @param flag
- */
- public void enableCleanup(boolean flag)
- {
- _cleanupFlag = flag;
- }
-
- /**
- * see clear data
- *
- * @return
- */
- public boolean isCleanupEnabled()
- {
- return _cleanupFlag;
- }
-
- /**
- *
- * @param flag
- */
- public void enableUpdateInPlace(boolean flag)
- {
- _updateInPlaceFlag = flag;
- }
-
- /**
- *
- * @return
- */
- public boolean isUpdateInPlaceEnabled()
- {
- return _updateInPlaceFlag;
- }
-
-
-
- /**
- *
- */
- public void setEmptyStatus()
- {
- setEmpty();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/PageCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/PageCache.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/PageCache.java
deleted file mode 100644
index 2219f58..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/caching/PageCache.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.caching;
-
-import java.lang.ref.SoftReference;
-import java.util.HashMap;
-import java.util.LinkedList;
-
-/**
- *
- *
- */
-public class PageCache
-{
-
- private static final int CLEANUP_THRESHOLD = 128;
- private static HashMap<Integer, LinkedList<SoftReference<byte[]>>> _pool;
-
- /**
- *
- */
- public static void init()
- {
- _pool = new HashMap<Integer, LinkedList<SoftReference<byte[]>>>();
- }
-
- /**
- *
- */
- public static void clear()
- {
- _pool = null;
- }
-
- /**
- *
- * @param data
- */
- public static void putPage( byte[] data )
- {
- //cleanup if too many different size lists
- if( _pool.size()>CLEANUP_THRESHOLD )
- _pool.clear();
-
- LinkedList<SoftReference<byte[]>> list = _pool.get( data.length );
- if( list==null ){
- list = new LinkedList<SoftReference<byte[]>>();
- _pool.put(data.length, list);
- }
- list.addLast(new SoftReference<byte[]>(data));
- }
-
- /**
- *
- * @param size
- * @return
- */
- public static byte[] getPage( int size )
- {
- LinkedList<SoftReference<byte[]>> list = _pool.get( size );
- if( list!=null ) {
- while( !list.isEmpty() ){
- SoftReference<byte[]> ref = list.removeFirst();
- byte[] tmp = ref.get();
- if( tmp!=null )
- return tmp;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContext.java
deleted file mode 100644
index 8bdfed8..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContext.java
+++ /dev/null
@@ -1,522 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.context;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLFrame;
-import com.ibm.bi.dml.debug.DMLProgramCounter;
-import com.ibm.bi.dml.debug.DebugState;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-
-public class ExecutionContext
-{
-
- //program reference (e.g., function repository)
- protected Program _prog = null;
-
- //symbol table
- protected LocalVariableMap _variables;
-
- //debugging (optional)
- protected DebugState _dbState = null;
-
- protected ExecutionContext()
- {
- //protected constructor to force use of ExecutionContextFactory
- this( true, null );
- }
-
- protected ExecutionContext(Program prog)
- {
- //protected constructor to force use of ExecutionContextFactory
- this( true, prog );
-
- }
-
- protected ExecutionContext(LocalVariableMap vars)
- {
- //protected constructor to force use of ExecutionContextFactory
- this( false, null);
- _variables = vars;
- }
-
- protected ExecutionContext( boolean allocateVariableMap, Program prog )
- {
- //protected constructor to force use of ExecutionContextFactory
- if( allocateVariableMap )
- _variables = new LocalVariableMap();
- else
- _variables = null;
- _prog = prog;
- if (DMLScript.ENABLE_DEBUG_MODE){
- _dbState = DebugState.getInstance();
- }
- }
-
- public Program getProgram(){
- return _prog;
- }
-
- public LocalVariableMap getVariables() {
- return _variables;
- }
-
- public void setVariables(LocalVariableMap vars) {
- _variables = vars;
- }
-
-
- /* -------------------------------------------------------
- * Methods to handle variables and associated data objects
- * -------------------------------------------------------
- */
-
- public Data getVariable(String name)
- {
- return _variables.get(name);
- }
-
- public void setVariable(String name, Data val)
- throws DMLRuntimeException
- {
- _variables.put(name, val);
- }
-
- public Data removeVariable(String name)
- {
- return _variables.remove(name);
- }
-
- public void setMetaData(String fname, MetaData md)
- throws DMLRuntimeException
- {
- _variables.get(fname).setMetaData(md);
- }
-
- public MetaData getMetaData(String varname)
- throws DMLRuntimeException
- {
- return _variables.get(varname).getMetaData();
- }
-
- public void removeMetaData(String varname)
- throws DMLRuntimeException
- {
- _variables.get(varname).removeMetaData();
- }
-
- public MatrixObject getMatrixObject(String varname)
- throws DMLRuntimeException
- {
- Data dat = getVariable(varname);
-
- //error handling if non existing or no matrix
- if( dat == null )
- throw new DMLRuntimeException("Variable '"+varname+"' does not exist in the symbol table.");
- if( !(dat instanceof MatrixObject) )
- throw new DMLRuntimeException("Variable '"+varname+"' is not a matrix.");
-
- return (MatrixObject) dat;
- }
-
- public MatrixCharacteristics getMatrixCharacteristics( String varname )
- throws DMLRuntimeException
- {
- MatrixDimensionsMetaData dims = (MatrixDimensionsMetaData) getMetaData(varname);
- return dims.getMatrixCharacteristics();
- }
-
- public MatrixBlock getMatrixInput(String varName)
- throws DMLRuntimeException
- {
- try {
- MatrixObject mobj = (MatrixObject) getVariable(varName);
- return mobj.acquireRead();
- } catch (CacheException e) {
- throw new DMLRuntimeException( e );
- }
- }
-
- public void releaseMatrixInput(String varName)
- throws DMLRuntimeException
- {
- try {
- ((MatrixObject)getVariable(varName)).release();
- } catch (CacheException e) {
- throw new DMLRuntimeException(e);
- }
- }
-
- public ScalarObject getScalarInput(String name, ValueType vt, boolean isLiteral)
- throws DMLRuntimeException
- {
- if ( isLiteral ) {
- switch (vt) {
- case INT:
- long intVal = UtilFunctions.parseToLong(name);
- IntObject intObj = new IntObject(intVal);
- return intObj;
- case DOUBLE:
- double doubleVal = Double.parseDouble(name);
- DoubleObject doubleObj = new DoubleObject(doubleVal);
- return doubleObj;
- case BOOLEAN:
- Boolean boolVal = Boolean.parseBoolean(name);
- BooleanObject boolObj = new BooleanObject(boolVal);
- return boolObj;
- case STRING:
- StringObject stringObj = new StringObject(name);
- return stringObj;
- default:
- throw new DMLRuntimeException("Unknown value type: " + vt + " for variable: " + name);
- }
- }
- else {
- Data obj = getVariable(name);
- if (obj == null) {
- throw new DMLRuntimeException("Unknown variable: " + name);
- }
- return (ScalarObject) obj;
- }
- }
-
-
- public void setScalarOutput(String varName, ScalarObject so)
- throws DMLRuntimeException
- {
- setVariable(varName, so);
- }
-
- public void setMatrixOutput(String varName, MatrixBlock outputData)
- throws DMLRuntimeException
- {
- MatrixObject sores = (MatrixObject) this.getVariable (varName);
-
- try
- {
- sores.acquireModify (outputData);
- sores.release();
-
- setVariable (varName, sores);
- }
- catch ( CacheException e ) {
- throw new DMLRuntimeException( e );
- }
- }
-
- /**
- *
- * @param varName
- * @param outputData
- * @param inplace
- * @throws DMLRuntimeException
- */
- public void setMatrixOutput(String varName, MatrixBlock outputData, boolean inplace)
- throws DMLRuntimeException
- {
- if( inplace ) //modify metadata to prevent output serialization
- {
- MatrixObject sores = (MatrixObject) this.getVariable (varName);
- sores.enableUpdateInPlace( true );
- }
-
- //default case
- setMatrixOutput(varName, outputData);
- }
-
- /**
- * Pin a given list of variables i.e., set the "clean up" state in
- * corresponding matrix objects, so that the cached data inside these
- * objects is not cleared and the corresponding HDFS files are not
- * deleted (through rmvar instructions).
- *
- * This is necessary for: function input variables, parfor result variables,
- * parfor shared inputs that are passed to functions.
- *
- * The function returns the OLD "clean up" state of matrix objects.
- */
- public HashMap<String,Boolean> pinVariables(ArrayList<String> varList)
- {
- //2-pass approach since multiple vars might refer to same matrix object
- HashMap<String, Boolean> varsState = new HashMap<String,Boolean>();
-
- //step 1) get current information
- for( String var : varList )
- {
- Data dat = _variables.get(var);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject)dat;
- varsState.put( var, mo.isCleanupEnabled() );
- //System.out.println("pre-pin "+var+" ("+mo.isCleanupEnabled()+")");
- }
- }
-
- //step 2) pin variables
- for( String var : varList )
- {
- Data dat = _variables.get(var);
- if( dat instanceof MatrixObject )
- {
- MatrixObject mo = (MatrixObject)dat;
- mo.enableCleanup(false);
- //System.out.println("pin "+var);
- }
- }
-
- return varsState;
- }
-
- /**
- * Unpin the a given list of variables by setting their "cleanup" status
- * to the values specified by <code>varsStats</code>.
- *
- * Typical usage:
- * <code>
- * oldStatus = pinVariables(varList);
- * ...
- * unpinVariables(varList, oldStatus);
- * </code>
- *
- * i.e., a call to unpinVariables() is preceded by pinVariables().
- */
- public void unpinVariables(ArrayList<String> varList, HashMap<String,Boolean> varsState)
- {
- for( String var : varList)
- {
- //System.out.println("unpin "+var+" ("+varsState.get(var)+")");
- Data dat = _variables.get(var);
- if( dat instanceof MatrixObject )
- ((MatrixObject)dat).enableCleanup(varsState.get(var));
- }
- }
-
- /**
- * NOTE: No order guaranteed, so keep same list for pin and unpin.
- *
- * @return
- */
- public ArrayList<String> getVarList()
- {
- ArrayList<String> varlist = new ArrayList<String>();
- varlist.addAll(_variables.keySet());
- return varlist;
- }
-
-
- /**
- *
- * @param mo
- * @throws CacheException
- * @throws IOException
- */
- public void cleanupMatrixObject(MatrixObject mo)
- throws DMLRuntimeException
- {
- try
- {
- if ( mo.isCleanupEnabled() )
- {
- //compute ref count only if matrix cleanup actually necessary
- if ( !getVariables().hasReferences(mo) ) {
- //clean cached data
- mo.clearData();
- if( mo.isFileExists() )
- {
- //clean hdfs data
- String fpath = mo.getFileName();
- if (fpath != null) {
- MapReduceTool.deleteFileIfExistOnHDFS(fpath);
- MapReduceTool.deleteFileIfExistOnHDFS(fpath + ".mtd");
- }
- }
- }
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
-
- ///////////////////////////////
- // Debug State Functionality
- ///////////////////////////////
-
- public void initDebugProgramCounters()
- {
- if (DMLScript.ENABLE_DEBUG_MODE){
- _dbState.pc = new DMLProgramCounter(DMLProgram.DEFAULT_NAMESPACE, "main", 0, 0); //initialize program counter (pc)
- _dbState.prevPC = new DMLProgramCounter(DMLProgram.DEFAULT_NAMESPACE, "main", 0, 0); //initialize previous pc
- }
- }
-
- /**
- *
- * @param index
- * @throws DMLRuntimeException
- */
- public void updateDebugState( int index ) throws DMLRuntimeException
- {
- if(DMLScript.ENABLE_DEBUG_MODE) {
- _dbState.getPC().setProgramBlockNumber(index);
- }
- }
-
- /**
- *
- * @param currInst
- * @throws DMLRuntimeException
- */
- public void updateDebugState( Instruction currInst ) throws DMLRuntimeException
- {
- if (DMLScript.ENABLE_DEBUG_MODE) {
- // New change so that shell doesnot seem like it is hanging while running MR job
- // Since UI cannot accept instructions when user is submitting the program
- _dbState.nextCommand = false;
- // Change to stop before first instruction of a given line
- //update current instruction ID and line number
- _dbState.getPC().setInstID(currInst.getInstID());
- _dbState.getPC().setLineNumber(currInst.getLineNum());
- // Change to stop before first instruction of a given line
- suspendIfAskedInDebugMode(currInst);
- }
- }
-
- /**
- *
- */
- public void clearDebugProgramCounters()
- {
- if(DMLScript.ENABLE_DEBUG_MODE) {
- _dbState.pc = null;
- }
- }
-
- public void handleDebugException( Exception ex )
- {
- _dbState.getDMLStackTrace(ex);
- _dbState.suspend = true;
- }
-
- public void handleDebugFunctionEntry( FunctionCallCPInstruction funCallInst ) throws DMLRuntimeException
- {
- //push caller frame into call stack
- _dbState.pushFrame(getVariables(), _dbState.getPC());
- //initialize pc for callee's frame
- _dbState.pc = new DMLProgramCounter(funCallInst.getNamespace(), funCallInst.getFunctionName(), 0, 0);
- }
-
- public void handleDebugFunctionExit( FunctionCallCPInstruction funCallInst )
- {
- //pop caller frame from call stack
- DMLFrame fr = _dbState.popFrame();
- //update pc to caller's frame
- _dbState.pc = fr.getPC();
- }
-
- public DebugState getDebugState() {
- return _dbState;
- }
-
- /**
- * This function should be called only if user has specified -debug option.
- * In this function, if the user has issued one of the step instructions or
- * has enabled suspend flag in previous instruction (through breakpoint),
- * then it will wait until user issues a new debugger command.
- * @param currInst
- * @param ec
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void suspendIfAskedInDebugMode(Instruction currInst ) throws DMLRuntimeException {
- if (!DMLScript.ENABLE_DEBUG_MODE) {
- System.err.println("ERROR: The function suspendIfAskedInDebugMode should not be called in non-debug mode.");
- }
- //check for stepping options
- if (!_dbState.suspend && _dbState.dbCommand != null) {
- if (_dbState.dbCommand.equalsIgnoreCase("step_instruction")) {
- System.out.format("Step instruction reached at %s.\n", _dbState.getPC().toString());
- _dbState.suspend = true;
- }
- else if (_dbState.dbCommand.equalsIgnoreCase("step_line") && _dbState.prevPC.getLineNumber() != currInst.getLineNum()
- && _dbState.prevPC.getLineNumber() != 0) {
- // Don't step into first instruction of first line
- // System.out.format("Step reached at %s.\n", this._prog.getPC().toString());
- System.out.format("Step reached at %s.\n", _dbState.getPC().toStringWithoutInstructionID());
- _dbState.suspend = true;
- }
- else if (_dbState.dbCommand.equalsIgnoreCase("step return") && currInst instanceof FunctionCallCPInstruction) {
- FunctionCallCPInstruction funCallInst = (FunctionCallCPInstruction) currInst;
- if (_dbState.dbCommandArg == null || funCallInst.getFunctionName().equalsIgnoreCase(_dbState.dbCommandArg)) {
- System.out.format("Step return reached at %s.\n", _dbState.getPC().toStringWithoutInstructionID());
- _dbState.suspend = true;
- }
- }
- }
- //check if runtime suspend signal is set
- if (_dbState.suspend) {
- //flush old commands and arguments
- _dbState.dbCommand = null;
- _dbState.dbCommandArg = null;
- //print current DML script source line
- if (currInst.getLineNum() != 0)
- _dbState.printDMLSourceLine(currInst.getLineNum());
- //save current symbol table
- _dbState.setVariables(this.getVariables());
- //send next command signal to debugger control module
- _dbState.nextCommand = true;
- //suspend runtime execution thread
- Thread.currentThread().suspend();
- //reset next command signal
- _dbState.nextCommand = false;
- }
- //reset runtime suspend signal
- _dbState.suspend = false;
- //update previous pc
- _dbState.prevPC.setFunctionName(_dbState.getPC().getFunctionName());
- _dbState.prevPC.setProgramBlockNumber(_dbState.getPC().getProgramBlockNumber());
- _dbState.prevPC.setLineNumber(currInst.getLineNum());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContextFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContextFactory.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContextFactory.java
deleted file mode 100644
index 62b517c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/context/ExecutionContextFactory.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.context;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-
-public class ExecutionContextFactory
-{
-
- /**
- *
- * @return
- */
- public static ExecutionContext createContext()
- {
- return createContext( null );
- }
-
- public static ExecutionContext createContext( Program prog )
- {
- return createContext(true, prog);
- }
-
- /**
- *
- * @param platform
- * @return
- */
- public static ExecutionContext createContext( boolean allocateVars, Program prog )
- {
- ExecutionContext ec = null;
-
- switch( DMLScript.rtplatform )
- {
- case SINGLE_NODE:
- case HADOOP:
- case HYBRID:
- ec = new ExecutionContext(allocateVars, prog);
- break;
-
- case SPARK:
- case HYBRID_SPARK:
- ec = new SparkExecutionContext(allocateVars, prog);
- break;
- }
-
- return ec;
- }
-}
[77/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/docs/mlcontext-programming-guide.md
----------------------------------------------------------------------
diff --git a/docs/mlcontext-programming-guide.md b/docs/mlcontext-programming-guide.md
index c7d415d..66caa3d 100644
--- a/docs/mlcontext-programming-guide.md
+++ b/docs/mlcontext-programming-guide.md
@@ -44,17 +44,17 @@ An `MLContext` object can be created by passing its constructor a reference to t
<div data-lang="Spark Shell" markdown="1">
{% highlight scala %}
-scala>import com.ibm.bi.dml.api.MLContext
-import com.ibm.bi.dml.api.MLContext
+scala>import org.apache.sysml.api.MLContext
+import org.apache.sysml.api.MLContext
scala> val ml = new MLContext(sc)
-ml: com.ibm.bi.dml.api.MLContext = com.ibm.bi.dml.api.MLContext@33e38c6b
+ml: org.apache.sysml.api.MLContext = org.apache.sysml.api.MLContext@33e38c6b
{% endhighlight %}
</div>
<div data-lang="Statements" markdown="1">
{% highlight scala %}
-import com.ibm.bi.dml.api.MLContext
+import org.apache.sysml.api.MLContext
val ml = new MLContext(sc)
{% endhighlight %}
</div>
@@ -125,27 +125,27 @@ an `MLOutput` object. The `getScalar()` method extracts a scalar value from a `D
<div data-lang="Spark Shell" markdown="1">
{% highlight scala %}
-scala> import com.ibm.bi.dml.api.MLOutput
-import com.ibm.bi.dml.api.MLOutput
+scala> import org.apache.sysml.api.MLOutput
+import org.apache.sysml.api.MLOutput
scala> def getScalar(outputs: MLOutput, symbol: String): Any =
| outputs.getDF(sqlContext, symbol).first()(1)
-getScalar: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Any
+getScalar: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Any
scala> def getScalarDouble(outputs: MLOutput, symbol: String): Double =
| getScalar(outputs, symbol).asInstanceOf[Double]
-getScalarDouble: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Double
+getScalarDouble: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Double
scala> def getScalarInt(outputs: MLOutput, symbol: String): Int =
| getScalarDouble(outputs, symbol).toInt
-getScalarInt: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Int
+getScalarInt: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Int
{% endhighlight %}
</div>
<div data-lang="Statements" markdown="1">
{% highlight scala %}
-import com.ibm.bi.dml.api.MLOutput
+import org.apache.sysml.api.MLOutput
def getScalar(outputs: MLOutput, symbol: String): Any =
outputs.getDF(sqlContext, symbol).first()(1)
def getScalarDouble(outputs: MLOutput, symbol: String): Double =
@@ -176,11 +176,11 @@ to convert the `DataFrame df` to a SystemML binary-block matrix, which is repres
<div data-lang="Spark Shell" markdown="1">
{% highlight scala %}
-scala> import com.ibm.bi.dml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
-import com.ibm.bi.dml.runtime.instructions.spark.utils.{RDDConverterUtilsExt=>RDDConverterUtils}
+scala> import org.apache.sysml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
+import org.apache.sysml.runtime.instructions.spark.utils.{RDDConverterUtilsExt=>RDDConverterUtils}
-scala> import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics
+scala> import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics
scala> val numRowsPerBlock = 1000
numRowsPerBlock: Int = 1000
@@ -189,18 +189,18 @@ scala> val numColsPerBlock = 1000
numColsPerBlock: Int = 1000
scala> val mc = new MatrixCharacteristics(numRows, numCols, numRowsPerBlock, numColsPerBlock)
-mc: com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics = [100000 x 1000, nnz=-1, blocks (1000 x 1000)]
+mc: org.apache.sysml.runtime.matrix.MatrixCharacteristics = [100000 x 1000, nnz=-1, blocks (1000 x 1000)]
scala> val sysMlMatrix = RDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc, false)
-sysMlMatrix: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@2bce3248
+sysMlMatrix: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@2bce3248
{% endhighlight %}
</div>
<div data-lang="Statements" markdown="1">
{% highlight scala %}
-import com.ibm.bi.dml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
val numRowsPerBlock = 1000
val numColsPerBlock = 1000
val mc = new MatrixCharacteristics(numRows, numCols, numRowsPerBlock, numColsPerBlock)
@@ -268,7 +268,7 @@ nargs: scala.collection.immutable.Map[String,String] = Map(Xin -> " ", Mout -> "
scala> val outputs = ml.execute("shape.dml", nargs)
15/10/12 16:29:15 WARN : Your hostname, derons-mbp.usca.ibm.com resolves to a loopback/non-reachable address: 127.0.0.1, but we couldn't find any external IP address!
15/10/12 16:29:15 WARN OptimizerUtils: Auto-disable multi-threaded text read for 'text' and 'csv' due to thread contention on JRE < 1.8 (java.version=1.7.0_80).
-outputs: com.ibm.bi.dml.api.MLOutput = com.ibm.bi.dml.api.MLOutput@4d424743
+outputs: org.apache.sysml.api.MLOutput = org.apache.sysml.api.MLOutput@4d424743
scala> val m = getScalarInt(outputs, "m")
m: Int = 100000
@@ -362,11 +362,11 @@ mean value of the matrix.
<div data-lang="Spark Shell" markdown="1">
{% highlight scala %}
-scala> import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes
+scala> import org.apache.sysml.runtime.matrix.data.MatrixIndexes
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes
-scala> import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock
+scala> import org.apache.sysml.runtime.matrix.data.MatrixBlock
+import org.apache.sysml.runtime.matrix.data.MatrixBlock
scala> import org.apache.spark.api.java.JavaPairRDD
import org.apache.spark.api.java.JavaPairRDD
@@ -383,15 +383,15 @@ scala> def minMaxMean(mat: JavaPairRDD[MatrixIndexes, MatrixBlock], rows: Int, c
| val meanOut = getScalarDouble(outputs, "meanOut")
| (minOut, maxOut, meanOut)
| }
-minMaxMean: (mat: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock], rows: Int, cols: Int, ml: com.ibm.bi.dml.api.MLContext)(Double, Double, Double)
+minMaxMean: (mat: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock], rows: Int, cols: Int, ml: org.apache.sysml.api.MLContext)(Double, Double, Double)
{% endhighlight %}
</div>
<div data-lang="Statements" markdown="1">
{% highlight scala %}
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes
+import org.apache.sysml.runtime.matrix.data.MatrixBlock
import org.apache.spark.api.java.JavaPairRDD
def minMaxMean(mat: JavaPairRDD[MatrixIndexes, MatrixBlock], rows: Int, cols: Int, ml: MLContext): (Double, Double, Double) = {
ml.reset()
@@ -452,7 +452,7 @@ to standard output.
{% highlight java %}
-package com.ibm.bi.dml;
+package org.apache.sysml;
import java.util.HashMap;
@@ -462,8 +462,8 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLOutput;
public class MLContextExample {
@@ -835,7 +835,7 @@ This cell contains helper methods to return `Double` and `Int` values from outpu
**Cell:**
{% highlight scala %}
// Helper functions
-import com.ibm.bi.dml.api.MLOutput
+import org.apache.sysml.api.MLOutput
def getScalar(outputs: MLOutput, symbol: String): Any =
outputs.getDF(sqlContext, symbol).first()(1)
@@ -849,10 +849,10 @@ def getScalarInt(outputs: MLOutput, symbol: String): Int =
**Output:**
{% highlight scala %}
-import com.ibm.bi.dml.api.MLOutput
-getScalar: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Any
-getScalarDouble: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Double
-getScalarInt: (outputs: com.ibm.bi.dml.api.MLOutput, symbol: String)Int
+import org.apache.sysml.api.MLOutput
+getScalar: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Any
+getScalarDouble: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Double
+getScalarInt: (outputs: org.apache.sysml.api.MLOutput, symbol: String)Int
{% endhighlight %}
@@ -867,9 +867,9 @@ and single-column `label` matrix, both represented by the
**Cell:**
{% highlight scala %}
// Imports
-import com.ibm.bi.dml.api.MLContext
-import com.ibm.bi.dml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.api.MLContext
+import org.apache.sysml.runtime.instructions.spark.utils.{RDDConverterUtilsExt => RDDConverterUtils}
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
// Create SystemML context
val ml = new MLContext(sc)
@@ -890,16 +890,16 @@ val cnt2 = y2.count()
**Output:**
{% highlight scala %}
-import com.ibm.bi.dml.api.MLContext
-import com.ibm.bi.dml.runtime.instructions.spark.utils.{RDDConverterUtilsExt=>RDDConverterUtils}
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics
-ml: com.ibm.bi.dml.api.MLContext = com.ibm.bi.dml.api.MLContext@38d59245
-mcX: com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics = [10000 x 1000, nnz=-1, blocks (1000 x 1000)]
-mcY: com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics = [10000 x 1, nnz=-1, blocks (1000 x 1000)]
-X: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@b5a86e3
-y: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@56377665
-X2: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@650f29d2
-y2: org.apache.spark.api.java.JavaPairRDD[com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes,com.ibm.bi.dml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@334857a8
+import org.apache.sysml.api.MLContext
+import org.apache.sysml.runtime.instructions.spark.utils.{RDDConverterUtilsExt=>RDDConverterUtils}
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics
+ml: org.apache.sysml.api.MLContext = org.apache.sysml.api.MLContext@38d59245
+mcX: org.apache.sysml.runtime.matrix.MatrixCharacteristics = [10000 x 1000, nnz=-1, blocks (1000 x 1000)]
+mcY: org.apache.sysml.runtime.matrix.MatrixCharacteristics = [10000 x 1, nnz=-1, blocks (1000 x 1000)]
+X: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@b5a86e3
+y: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@56377665
+X2: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@650f29d2
+y2: org.apache.spark.api.java.JavaPairRDD[org.apache.sysml.runtime.matrix.data.MatrixIndexes,org.apache.sysml.runtime.matrix.data.MatrixBlock] = org.apache.spark.api.java.JavaPairRDD@334857a8
cnt1: Long = 10
cnt2: Long = 10
{% endhighlight %}
@@ -936,7 +936,7 @@ val trainingTimePerIter = trainingTime / iters
**Output:**
{% highlight scala %}
start: Long = 1444672090620
-outputs: com.ibm.bi.dml.api.MLOutput = com.ibm.bi.dml.api.MLOutput@5d2c22d0
+outputs: org.apache.sysml.api.MLOutput = org.apache.sysml.api.MLOutput@5d2c22d0
trainingTime: Double = 1.176
B: org.apache.spark.sql.DataFrame = [C1: double]
r2: Double = 0.9677079547216473
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/docs/quick-start-guide.md
----------------------------------------------------------------------
diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md
index b01aea1..1a7935b 100644
--- a/docs/quick-start-guide.md
+++ b/docs/quick-start-guide.md
@@ -357,7 +357,7 @@ If you encounter a `"java.lang.OutOfMemoryError"` you can edit the invocation
script (`runStandaloneSystemML.sh` or `runStandaloneSystemML.bat`) to increase
the memory available to the JVM, i.e:
- java -Xmx16g -Xms4g -Xmn1g -cp ${CLASSPATH} com.ibm.bi.dml.api.DMLScript \
+ java -Xmx16g -Xms4g -Xmn1g -cp ${CLASSPATH} org.apache.sysml.api.DMLScript \
-f ${SCRIPT_FILE} -exec singlenode -config=SystemML-config.xml \
$@
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 23c7273..ebbba39 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@
<goal>java</goal>
</goals>
<configuration>
- <mainClass>com.ibm.bi.dml.utils.InstallDependencyForIntegrationTests</mainClass>
+ <mainClass>org.apache.sysml.utils.InstallDependencyForIntegrationTests</mainClass>
<arguments>
<argument>${skipTests}</argument>
</arguments>
@@ -262,7 +262,7 @@
<!-- </manifest> -->
<manifestEntries>
<Build-Timestamp>${maven.build.timestamp}</Build-Timestamp>
- <Main-Class>com.ibm.bi.dml.api.DMLScript</Main-Class>
+ <Main-Class>org.apache.sysml.api.DMLScript</Main-Class>
</manifestEntries>
</archive>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/scripts/algorithms/decision-tree.dml
----------------------------------------------------------------------
diff --git a/scripts/algorithms/decision-tree.dml b/scripts/algorithms/decision-tree.dml
index 6dfe688..5c90976 100644
--- a/scripts/algorithms/decision-tree.dml
+++ b/scripts/algorithms/decision-tree.dml
@@ -63,7 +63,7 @@
# External function for binning
binning = externalFunction(Matrix[Double] A, Integer binsize, Integer numbins) return (Matrix[Double] B, Integer numbinsdef)
- implemented in (classname="com.ibm.bi.dml.udf.lib.BinningWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.BinningWrapper",exectype="mem")
# Default values of some parameters
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/scripts/algorithms/random-forest.dml
----------------------------------------------------------------------
diff --git a/scripts/algorithms/random-forest.dml b/scripts/algorithms/random-forest.dml
index df68869..4bf0250 100644
--- a/scripts/algorithms/random-forest.dml
+++ b/scripts/algorithms/random-forest.dml
@@ -67,7 +67,7 @@
# External function for binning
binning = externalFunction(Matrix[Double] A, Integer binsize, Integer numbins) return (Matrix[Double] B, Integer numbinsdef)
- implemented in (classname="com.ibm.bi.dml.udf.lib.BinningWrapper",exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.BinningWrapper",exectype="mem")
# Default values of some parameters
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/scripts/staging/Lanczos.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/Lanczos.dml b/scripts/staging/Lanczos.dml
index 6d4b2ab..e432ab1 100644
--- a/scripts/staging/Lanczos.dml
+++ b/scripts/staging/Lanczos.dml
@@ -3,7 +3,7 @@
eigen = externalFunction(Matrix[Double] A)
return(Matrix[Double] eval, Matrix[Double] evec)
- implemented in (classname="com.ibm.bi.dml.udf.lib.EigenWrapper", exectype="mem")
+ implemented in (classname="org.apache.sysml.udf.lib.EigenWrapper", exectype="mem")
A = read($1); # input data, must be a square matrix
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/DMLException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLException.java b/src/main/java/org/apache/sysml/api/DMLException.java
index 078c067..92d350d 100644
--- a/src/main/java/org/apache/sysml/api/DMLException.java
+++ b/src/main/java/org/apache/sysml/api/DMLException.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
/**
* <p>Exception occurring in the DML framework.</p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 0b2d59e..0c21be9 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -45,46 +45,46 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.xml.sax.SAXException;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.debug.DMLDebugger;
-import com.ibm.bi.dml.debug.DMLDebuggerException;
-import com.ibm.bi.dml.debug.DMLDebuggerProgramInfo;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.hops.globalopt.GlobalOptimizerWrapper;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.matrix.CleanupMR;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRConfigurationNames;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Explain.ExplainCounts;
-import com.ibm.bi.dml.utils.Explain.ExplainType;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-// import com.ibm.bi.dml.utils.visualize.DotGraph;
-import com.ibm.bi.dml.yarn.DMLYarnClientProxy;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.debug.DMLDebugger;
+import org.apache.sysml.debug.DMLDebuggerException;
+import org.apache.sysml.debug.DMLDebuggerProgramInfo;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.OptimizerUtils.OptimizationLevel;
+import org.apache.sysml.hops.globalopt.GlobalOptimizerWrapper;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLScriptException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler;
+import org.apache.sysml.runtime.matrix.CleanupMR;
+import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Explain;
+import org.apache.sysml.utils.Explain.ExplainCounts;
+import org.apache.sysml.utils.Explain.ExplainType;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
+// import org.apache.sysml.utils.visualize.DotGraph;
+import org.apache.sysml.yarn.DMLYarnClientProxy;
public class DMLScript
@@ -536,10 +536,10 @@ public class DMLScript
if (debug != null){
if (debug.equalsIgnoreCase("debug")){
- Logger.getLogger("com.ibm.bi.dml").setLevel((Level) Level.DEBUG);
+ Logger.getLogger("org.apache.sysml").setLevel((Level) Level.DEBUG);
}
else if (debug.equalsIgnoreCase("trace")){
- Logger.getLogger("com.ibm.bi.dml").setLevel((Level) Level.TRACE);
+ Logger.getLogger("org.apache.sysml").setLevel((Level) Level.TRACE);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/MLBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLBlock.java b/src/main/java/org/apache/sysml/api/MLBlock.java
index b68f1ad..32ce5f3 100644
--- a/src/main/java/org/apache/sysml/api/MLBlock.java
+++ b/src/main/java/org/apache/sysml/api/MLBlock.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.math.BigDecimal;
import java.sql.Date;
@@ -27,8 +27,8 @@ import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import scala.collection.Seq;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 39c6384..0099d8a 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.io.IOException;
@@ -32,53 +32,53 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.rdd.RDD;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.api.jmlc.JMLCUtils;
-import com.ibm.bi.dml.api.monitoring.SparkMonitoringUtil;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.hops.globalopt.GlobalOptimizerWrapper;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteRemovePersistentReadWrite;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDProperties;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ConvertStringToLongTextPair;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockPairFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyTextInputFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.SparkListener;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.utils.Explain.ExplainCounts;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.jmlc.JMLCUtils;
+import org.apache.sysml.api.monitoring.SparkMonitoringUtil;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.OptimizerUtils.OptimizationLevel;
+import org.apache.sysml.hops.globalopt.GlobalOptimizerWrapper;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.IntIdentifier;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.StringIdentifier;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.data.RDDProperties;
+import org.apache.sysml.runtime.instructions.spark.functions.ConvertStringToLongTextPair;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockPairFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyTextInputFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.SparkListener;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.utils.Explain;
+import org.apache.sysml.utils.Statistics;
+import org.apache.sysml.utils.Explain.ExplainCounts;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
@@ -89,7 +89,7 @@ import org.apache.spark.sql.SQLContext;
* <p>
* Typical usage for MLContext is as follows:
* <pre><code>
- * scala> import com.ibm.bi.dml.api.MLContext
+ * scala> import org.apache.sysml.api.MLContext
* </code></pre>
* <p>
* Create input DataFrame from CSV file and potentially perform some feature transformation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/MLContextProxy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContextProxy.java b/src/main/java/org/apache/sysml/api/MLContextProxy.java
index ad32c5f..d7de98d 100644
--- a/src/main/java/org/apache/sysml/api/MLContextProxy.java
+++ b/src/main/java/org/apache/sysml/api/MLContextProxy.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.monitoring.Location;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.api.monitoring.Location;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
/**
* The purpose of this proxy is to shield systemml internals from direct access to MLContext
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/MLMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLMatrix.java b/src/main/java/org/apache/sysml/api/MLMatrix.java
index 541f094..00f56c4 100644
--- a/src/main/java/org/apache/sysml/api/MLMatrix.java
+++ b/src/main/java/org/apache/sysml/api/MLMatrix.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.io.IOException;
import java.util.List;
@@ -32,14 +32,14 @@ import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMIMBFromRow;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMLBlock;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.functions.GetMIMBFromRow;
+import org.apache.sysml.runtime.instructions.spark.functions.GetMLBlock;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Experimental API: Might be discontinued in future release
@@ -56,7 +56,7 @@ import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
* Also, note that this library is not thread-safe. The operator precedence is not exactly same as DML (as the precedence is
* enforced by scala compiler), so please use appropriate brackets to enforce precedence.
- import com.ibm.bi.dml.api.{MLContext, MLMatrix}
+ import org.apache.sysml.api.{MLContext, MLMatrix}
val ml = new MLContext(sc)
val mat1 = ml.read(sqlContext, "V_small.csv", "csv")
val mat2 = ml.read(sqlContext, "W_small.mtx", "binary")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/MLOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLOutput.java b/src/main/java/org/apache/sysml/api/MLOutput.java
index 82f74a0..299ce50 100644
--- a/src/main/java/org/apache/sysml/api/MLOutput.java
+++ b/src/main/java/org/apache/sysml/api/MLOutput.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.api;
+package org.apache.sysml.api;
import java.util.ArrayList;
import java.util.HashMap;
@@ -38,13 +38,13 @@ import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMLBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.functions.GetMLBlock;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* This is a simple container object that returns the output of execute from MLContext
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 57d389f..2c72ee1 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.api.jmlc;
+package org.apache.sysml.api.jmlc;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -28,26 +28,26 @@ import java.util.HashMap;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteRemovePersistentReadWrite;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.io.MatrixReaderFactory;
-import com.ibm.bi.dml.runtime.io.ReaderTextCell;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.io.ReaderTextCell;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* JMLC (Java Machine Learning Connector) API:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java b/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
index 3cf3e18..57f4581 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.api.jmlc;
+package org.apache.sysml.api.jmlc;
import java.util.ArrayList;
import java.util.Map;
import java.util.Map.Entry;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
public class JMLCUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index c3d4555..071ccf3 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -15,30 +15,30 @@
*
*/
-package com.ibm.bi.dml.api.jmlc;
+package org.apache.sysml.api.jmlc;
import java.util.HashSet;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* JMLC (Java Machine Learning Connector) API:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
index dedb08f..d226030 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.api.jmlc;
+package org.apache.sysml.api.jmlc;
import java.util.HashMap;
import java.util.Set;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* JMLC (Java Machine Learning Connector) API:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java b/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
index 8803d76..523c77a 100644
--- a/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
+++ b/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.api.ml;
+package org.apache.sysml.api.ml;
import java.io.File;
import java.io.IOException;
@@ -18,17 +18,17 @@ import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
-import com.ibm.bi.dml.api.ml.LogisticRegressionModel;
-import com.ibm.bi.dml.api.ml.functions.ConvertSingleColumnToString;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLOutput;
+import org.apache.sysml.api.ml.LogisticRegressionModel;
+import org.apache.sysml.api.ml.functions.ConvertSingleColumnToString;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
*
@@ -46,7 +46,7 @@ import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
* <pre><code>
* // Code to demonstrate usage of pipeline
* import org.apache.spark.ml.Pipeline
- * import com.ibm.bi.dml.api.ml.LogisticRegression
+ * import org.apache.sysml.api.ml.LogisticRegression
* import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
* import org.apache.spark.mllib.linalg.Vector
* case class LabeledDocument(id: Long, text: String, label: Double)
@@ -78,7 +78,7 @@ import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
*
* // Code to demonstrate usage of cross-validation
* import org.apache.spark.ml.Pipeline
- * import com.ibm.bi.dml.api.ml.LogisticRegression
+ * import org.apache.sysml.api.ml.LogisticRegression
* import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
* import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
* import org.apache.spark.ml.tuning.{ParamGridBuilder, CrossValidator}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java b/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
index 590372e..6880c2d 100644
--- a/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
+++ b/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.api.ml;
+package org.apache.sysml.api.ml;
import java.io.File;
import java.io.IOException;
@@ -16,15 +16,15 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.api.MLContext;
-import com.ibm.bi.dml.api.MLOutput;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.MLContext;
+import org.apache.sysml.api.MLOutput;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
public class LogisticRegressionModel extends ProbabilisticClassificationModel<Vector, LogisticRegressionModel> {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/ml/functions/ConvertSingleColumnToString.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/functions/ConvertSingleColumnToString.java b/src/main/java/org/apache/sysml/api/ml/functions/ConvertSingleColumnToString.java
index 32844de..38f8d27 100644
--- a/src/main/java/org/apache/sysml/api/ml/functions/ConvertSingleColumnToString.java
+++ b/src/main/java/org/apache/sysml/api/ml/functions/ConvertSingleColumnToString.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.api.ml.functions;
+package org.apache.sysml.api.ml.functions;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Row;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/ml/functions/ConvertVectorToDouble.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/functions/ConvertVectorToDouble.java b/src/main/java/org/apache/sysml/api/ml/functions/ConvertVectorToDouble.java
index e9c77ba..f1152d1 100644
--- a/src/main/java/org/apache/sysml/api/ml/functions/ConvertVectorToDouble.java
+++ b/src/main/java/org/apache/sysml/api/ml/functions/ConvertVectorToDouble.java
@@ -1,4 +1,4 @@
-package com.ibm.bi.dml.api.ml.functions;
+package org.apache.sysml.api.ml.functions;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Row;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/monitoring/InstructionComparator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/monitoring/InstructionComparator.java b/src/main/java/org/apache/sysml/api/monitoring/InstructionComparator.java
index 906ebb4..539ab9b 100644
--- a/src/main/java/org/apache/sysml/api/monitoring/InstructionComparator.java
+++ b/src/main/java/org/apache/sysml/api/monitoring/InstructionComparator.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.api.monitoring;
+package org.apache.sysml.api.monitoring;
import java.util.Comparator;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/monitoring/Location.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/monitoring/Location.java b/src/main/java/org/apache/sysml/api/monitoring/Location.java
index 3cc2957..efab6b4 100644
--- a/src/main/java/org/apache/sysml/api/monitoring/Location.java
+++ b/src/main/java/org/apache/sysml/api/monitoring/Location.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.api.monitoring;
+package org.apache.sysml.api.monitoring;
public class Location implements Comparable<Location> {
public int beginLine;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
index 9b4996b..897b18f 100644
--- a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
+++ b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
@@ -14,7 +14,7 @@
* limitations under the License.
*
*/
-package com.ibm.bi.dml.api.monitoring;
+package org.apache.sysml.api.monitoring;
import java.io.BufferedWriter;
import java.io.FileWriter;
@@ -31,11 +31,11 @@ import scala.xml.Node;
import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.SparkListener;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.runtime.instructions.spark.functions.SparkListener;
/**
* Usage guide:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/api/python/SystemML.py
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/python/SystemML.py b/src/main/java/org/apache/sysml/api/python/SystemML.py
index a4b22ff..e02d185 100644
--- a/src/main/java/org/apache/sysml/api/python/SystemML.py
+++ b/src/main/java/org/apache/sysml/api/python/SystemML.py
@@ -36,7 +36,7 @@ class MLContext(object):
monitorPerformance = (args[0] if len(args) > 0 else False)
setForcedSparkExecType = (args[1] if len(args) > 1 else False)
self.sc = sc
- self.ml = sc._jvm.com.ibm.bi.dml.api.MLContext(sc._jsc, monitorPerformance, setForcedSparkExecType)
+ self.ml = sc._jvm.org.apache.sysml.api.MLContext(sc._jsc, monitorPerformance, setForcedSparkExecType)
except Py4JError:
traceback.print_exc()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
index a8fa5f6..bdf474a 100644
--- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
+++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.conf;
+package org.apache.sysml.conf;
import org.apache.hadoop.mapred.JobConf;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 2b54f7b..cd26953 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.conf;
+package org.apache.sysml.conf;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
@@ -41,9 +41,9 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.LocalFileUtils;
public class DMLConfig
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java b/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
index 80b0447..13080c1 100644
--- a/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
+++ b/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.util.TreeMap;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDebugger.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebugger.java b/src/main/java/org/apache/sysml/debug/DMLDebugger.java
index 33d8116..7bedbb4 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebugger.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebugger.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.io.PrintStream;
import java.util.HashMap;
@@ -23,10 +23,10 @@ import java.util.HashMap;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.lang.math.IntRange;
-import com.ibm.bi.dml.debug.DMLDebuggerFunctions;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
+import org.apache.sysml.debug.DMLDebuggerFunctions;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
/**
* This class implements a debugger control module for DML scripts.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDebuggerException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerException.java b/src/main/java/org/apache/sysml/debug/DMLDebuggerException.java
index 132a5c7..b257a52 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerException.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerException.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
-import com.ibm.bi.dml.api.DMLException;
+import org.apache.sysml.api.DMLException;
/**
* This exception should be thrown to flag debugger errors.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java b/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
index 94ce010..637074e 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.util.ArrayList;
import java.util.Map.Entry;
@@ -24,24 +24,24 @@ import java.util.TreeMap;
import org.apache.commons.lang.math.IntRange;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
public class DMLDebuggerFunctions {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java b/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
index b2bab75..420d7f0 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.io.BufferedReader;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDebuggerProgramInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerProgramInfo.java b/src/main/java/org/apache/sysml/debug/DMLDebuggerProgramInfo.java
index f90ca49..431a6df 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerProgramInfo.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerProgramInfo.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
/**
* This class contains the parsed and compiled DML script w/ hops, lops and runtime program.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDisassemble.java b/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
index 1e1838a..cb38d03 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLFrame.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLFrame.java b/src/main/java/org/apache/sysml/debug/DMLFrame.java
index a43f269..0a3522d 100644
--- a/src/main/java/org/apache/sysml/debug/DMLFrame.java
+++ b/src/main/java/org/apache/sysml/debug/DMLFrame.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
public class DMLFrame {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DMLProgramCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLProgramCounter.java b/src/main/java/org/apache/sysml/debug/DMLProgramCounter.java
index 8f5dc80..7b83e49 100644
--- a/src/main/java/org/apache/sysml/debug/DMLProgramCounter.java
+++ b/src/main/java/org/apache/sysml/debug/DMLProgramCounter.java
@@ -16,9 +16,9 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
-import com.ibm.bi.dml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLProgram;
public class DMLProgramCounter {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/debug/DebugState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DebugState.java b/src/main/java/org/apache/sysml/debug/DebugState.java
index b7330b9..e37f6b3 100644
--- a/src/main/java/org/apache/sysml/debug/DebugState.java
+++ b/src/main/java/org/apache/sysml/debug/DebugState.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.debug;
+package org.apache.sysml.debug;
import java.util.Stack;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
public class DebugState
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index 005744f..30ca1e9 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -15,41 +15,41 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.MMCJ;
-import com.ibm.bi.dml.lops.MMRJ;
-import com.ibm.bi.dml.lops.MMTSJ;
-import com.ibm.bi.dml.lops.MMCJ.MMCJType;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.lops.MMZip;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.lops.PMapMult;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.Transform;
-import com.ibm.bi.dml.lops.Transform.OperationTypes;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Binary;
+import org.apache.sysml.lops.DataPartition;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.MMCJ;
+import org.apache.sysml.lops.MMRJ;
+import org.apache.sysml.lops.MMTSJ;
+import org.apache.sysml.lops.MMCJ.MMCJType;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.lops.MMZip;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.lops.PMMJ;
+import org.apache.sysml.lops.PMapMult;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.Transform;
+import org.apache.sysml.lops.Transform.OperationTypes;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
/* Aggregate binary (cell operations): Sum (aij + bij)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index a0be584..b1463e9 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -15,27 +15,27 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes;
-import com.ibm.bi.dml.lops.TernaryAggregate;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.Aggregate.OperationTypes;
+import org.apache.sysml.lops.Binary;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.PartialAggregate;
+import org.apache.sysml.lops.PartialAggregate.DirectionTypes;
+import org.apache.sysml.lops.TernaryAggregate;
+import org.apache.sysml.lops.UAggOuterChain;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
/* Aggregate unary (cell) operation: Sum (aij), col_sum, row_sum
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index 9d6d6fb..a39a2e8 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -15,41 +15,41 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.AppendGAlignedSP;
-import com.ibm.bi.dml.lops.AppendM;
-import com.ibm.bi.dml.lops.AppendCP;
-import com.ibm.bi.dml.lops.AppendG;
-import com.ibm.bi.dml.lops.AppendR;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.BinaryScalar;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.BinaryUAggChain;
-import com.ibm.bi.dml.lops.CentralMoment;
-import com.ibm.bi.dml.lops.CoVariance;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.CombineUnary;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.CombineBinary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Aggregate;
+import org.apache.sysml.lops.AppendGAlignedSP;
+import org.apache.sysml.lops.AppendM;
+import org.apache.sysml.lops.AppendCP;
+import org.apache.sysml.lops.AppendG;
+import org.apache.sysml.lops.AppendR;
+import org.apache.sysml.lops.Binary;
+import org.apache.sysml.lops.BinaryScalar;
+import org.apache.sysml.lops.BinaryM;
+import org.apache.sysml.lops.BinaryUAggChain;
+import org.apache.sysml.lops.CentralMoment;
+import org.apache.sysml.lops.CoVariance;
+import org.apache.sysml.lops.CombineBinary;
+import org.apache.sysml.lops.CombineUnary;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.DataPartition;
+import org.apache.sysml.lops.Group;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.PartialAggregate;
+import org.apache.sysml.lops.PickByCount;
+import org.apache.sysml.lops.RepMat;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.lops.Unary;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.CombineBinary.OperationTypes;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
/* Binary (cell operations): aij + bij
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java
index 90d5ec0..dd40f8c 100644
--- a/src/main/java/org/apache/sysml/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.hops.Hop.MultiThreadedHop;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java
index 3975ed9..943cfba 100644
--- a/src/main/java/org/apache/sysml/hops/DataOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataOp.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
+import org.apache.sysml.lops.Data;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.util.LocalFileUtils;
import java.util.HashMap;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/FunctionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/FunctionOp.java b/src/main/java/org/apache/sysml/hops/FunctionOp.java
index 3fe3c61..05f30ae 100644
--- a/src/main/java/org/apache/sysml/hops/FunctionOp.java
+++ b/src/main/java/org/apache/sysml/hops/FunctionOp.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.hops;
+package org.apache.sysml.hops;
import java.util.ArrayList;
-import com.ibm.bi.dml.lops.FunctionCallCP;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
+import org.apache.sysml.lops.FunctionCallCP;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
/**
* This FunctionOp represents the call to a DML-bodied or external function.
[44/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/AggUnaryOp.java b/src/main/java/com/ibm/bi/dml/hops/AggUnaryOp.java
deleted file mode 100644
index a0be584..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/AggUnaryOp.java
+++ /dev/null
@@ -1,735 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PartialAggregate.DirectionTypes;
-import com.ibm.bi.dml.lops.TernaryAggregate;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-
-/* Aggregate unary (cell) operation: Sum (aij), col_sum, row_sum
- * Properties:
- * Symbol: +, min, max, ...
- * 1 Operand
- *
- * Semantic: generate indices, align, aggregate
- */
-
-public class AggUnaryOp extends Hop implements MultiThreadedHop
-{
-
- private static final boolean ALLOW_UNARYAGG_WO_FINAL_AGG = true;
-
- private AggOp _op;
- private Direction _direction;
-
- private int _maxNumThreads = -1; //-1 for unlimited
-
- private AggUnaryOp() {
- //default constructor for clone
- }
-
- public AggUnaryOp(String l, DataType dt, ValueType vt, AggOp o, Direction idx, Hop inp)
- {
- super(l, dt, vt);
- _op = o;
- _direction = idx;
- getInput().add(0, inp);
- inp.getParent().add(this);
- }
-
- public AggOp getOp()
- {
- return _op;
- }
-
- public void setOp(AggOp op)
- {
- _op = op;
- }
-
- public Direction getDirection()
- {
- return _direction;
- }
-
- public void setDirection(Direction direction)
- {
- _direction = direction;
- }
-
- @Override
- public void setMaxNumThreads( int k ) {
- _maxNumThreads = k;
- }
-
- @Override
- public int getMaxNumThreads() {
- return _maxNumThreads;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- try
- {
- ExecType et = optFindExecType();
- Hop input = getInput().get(0);
-
- if ( et == ExecType.CP )
- {
- Lop agg1 = null;
- if( isTernaryAggregateRewriteApplicable() ) {
- agg1 = constructLopsTernaryAggregateRewrite(et);
- }
- else if( isUnaryAggregateOuterCPRewriteApplicable() )
- {
- OperationTypes op = HopsAgg2Lops.get(_op);
- DirectionTypes dir = HopsDirection2Lops.get(_direction);
-
- BinaryOp binput = (BinaryOp)getInput().get(0);
- agg1 = new UAggOuterChain( binput.getInput().get(0).constructLops(),
- binput.getInput().get(1).constructLops(), op, dir,
- HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.CP);
- PartialAggregate.setDimensionsBasedOnDirection(agg1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
-
- if (getDataType() == DataType.SCALAR) {
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
-
- }
- else { //general case
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- agg1 = new PartialAggregate(input.constructLops(),
- HopsAgg2Lops.get(_op), HopsDirection2Lops.get(_direction), getDataType(),getValueType(), et, k);
- }
-
- setOutputDimensions(agg1);
- setLineNumbers(agg1);
- setLops(agg1);
-
- if (getDataType() == DataType.SCALAR) {
- agg1.getOutputParameters().setDimensions(1, 1, getRowsInBlock(), getColsInBlock(), getNnz());
- }
- }
- else if( et == ExecType.MR )
- {
- OperationTypes op = HopsAgg2Lops.get(_op);
- DirectionTypes dir = HopsDirection2Lops.get(_direction);
-
- //unary aggregate operation
- Lop transform1 = null;
- if( isUnaryAggregateOuterRewriteApplicable() )
- {
- BinaryOp binput = (BinaryOp)getInput().get(0);
- transform1 = new UAggOuterChain( binput.getInput().get(0).constructLops(),
- binput.getInput().get(1).constructLops(), op, dir,
- HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.MR);
- PartialAggregate.setDimensionsBasedOnDirection(transform1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
- }
- else //default
- {
- transform1 = new PartialAggregate(input.constructLops(), op, dir, DataType.MATRIX, getValueType());
- ((PartialAggregate) transform1).setDimensionsBasedOnDirection(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock());
- }
- setLineNumbers(transform1);
-
- //aggregation if required
- Lop aggregate = null;
- Group group1 = null;
- Aggregate agg1 = null;
- if( requiresAggregation(input, _direction) || transform1 instanceof UAggOuterChain )
- {
- group1 = new Group(transform1, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
- group1.getOutputParameters().setDimensions(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), getNnz());
- setLineNumbers(group1);
-
- agg1 = new Aggregate(group1, HopsAgg2Lops.get(_op), DataType.MATRIX, getValueType(), et);
- agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), getNnz());
- agg1.setupCorrectionLocation(PartialAggregate.getCorrectionLocation(op,dir));
- setLineNumbers(agg1);
-
- aggregate = agg1;
- }
- else
- {
- ((PartialAggregate) transform1).setDropCorrection();
- aggregate = transform1;
- }
-
- setLops(aggregate);
-
- //cast if required
- if (getDataType() == DataType.SCALAR) {
-
- // Set the dimensions of PartialAggregate LOP based on the
- // direction in which aggregation is performed
- PartialAggregate.setDimensionsBasedOnDirection(transform1, input.getDim1(), input.getDim2(),
- input.getRowsInBlock(), input.getColsInBlock(), dir);
-
- if( group1 != null && agg1 != null ) { //if aggregation required
- group1.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(),
- input.getRowsInBlock(), input.getColsInBlock(), getNnz());
- agg1.getOutputParameters().setDimensions(1, 1,
- input.getRowsInBlock(), input.getColsInBlock(), getNnz());
- }
-
- UnaryCP unary1 = new UnaryCP(
- aggregate, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- }
- else if( et == ExecType.SPARK )
- {
- OperationTypes op = HopsAgg2Lops.get(_op);
- DirectionTypes dir = HopsDirection2Lops.get(_direction);
-
- //unary aggregate
- if( isTernaryAggregateRewriteApplicable() )
- {
- Lop aggregate = constructLopsTernaryAggregateRewrite(et);
- setOutputDimensions(aggregate); //0x0 (scalar)
- setLineNumbers(aggregate);
- setLops(aggregate);
- }
- else if( isUnaryAggregateOuterSPRewriteApplicable() )
- {
- BinaryOp binput = (BinaryOp)getInput().get(0);
- Lop transform1 = new UAggOuterChain( binput.getInput().get(0).constructLops(),
- binput.getInput().get(1).constructLops(), op, dir,
- HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.SPARK);
- PartialAggregate.setDimensionsBasedOnDirection(transform1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
- setLineNumbers(transform1);
- setLops(transform1);
-
- if (getDataType() == DataType.SCALAR) {
- UnaryCP unary1 = new UnaryCP(transform1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
-
- }
- else //default
- {
- boolean needAgg = requiresAggregation(input, _direction);
- SparkAggType aggtype = getSparkUnaryAggregationType(needAgg);
-
- PartialAggregate aggregate = new PartialAggregate(input.constructLops(),
- HopsAgg2Lops.get(_op), HopsDirection2Lops.get(_direction), DataType.MATRIX, getValueType(), aggtype, et);
- aggregate.setDimensionsBasedOnDirection(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock());
- setLineNumbers(aggregate);
- setLops(aggregate);
-
- if (getDataType() == DataType.SCALAR) {
- UnaryCP unary1 = new UnaryCP(aggregate, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- }
- }
- }
- catch (Exception e) {
- throw new HopsException(this.printErrorLocation() + "In AggUnary Hop, error constructing Lops " , e);
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- //return created lops
- return getLops();
- }
-
-
-
- @Override
- public String getOpString() {
- //ua - unary aggregate, for consistency with runtime
- String s = "ua(" +
- HopsAgg2String.get(_op) +
- HopsDirection2String.get(_direction) + ")";
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + _op);
- LOG.debug(" Direction: " + _direction);
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- //default: no additional memory required
- double val = 0;
-
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
-
- switch( _op ) //see MatrixAggLib for runtime operations
- {
- case MAX:
- case MIN:
- //worst-case: column-wise, sparse (temp int count arrays)
- if( _direction == Direction.Col )
- val = dim2 * OptimizerUtils.INT_SIZE;
- break;
- case SUM:
- case SUM_SQ:
- //worst-case correction LASTROW / LASTCOLUMN
- if( _direction == Direction.Col ) //(potentially sparse)
- val = OptimizerUtils.estimateSizeExactSparsity(1, dim2, sparsity);
- else if( _direction == Direction.Row ) //(always dense)
- val = OptimizerUtils.estimateSizeExactSparsity(dim1, 1, 1.0);
- break;
- case MEAN:
- //worst-case correction LASTTWOROWS / LASTTWOCOLUMNS
- if( _direction == Direction.Col ) //(potentially sparse)
- val = OptimizerUtils.estimateSizeExactSparsity(2, dim2, sparsity);
- else if( _direction == Direction.Row ) //(always dense)
- val = OptimizerUtils.estimateSizeExactSparsity(dim1, 2, 1.0);
- break;
- case MAXINDEX:
- case MININDEX:
- Hop hop = getInput().get(0);
- if(isUnaryAggregateOuterCPRewriteApplicable())
- val = 3 * OptimizerUtils.estimateSizeExactSparsity(1, hop._dim2, 1.0);
- else
- //worst-case correction LASTCOLUMN
- val = OptimizerUtils.estimateSizeExactSparsity(dim1, 1, 1.0);
- break;
- default:
- //no intermediate memory consumption
- val = 0;
- }
-
- return val;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- Hop input = getInput().get(0);
- MatrixCharacteristics mc = memo.getAllInputStats(input);
- if( _direction == Direction.Col && mc.colsKnown() )
- ret = new long[]{1, mc.getCols(), -1};
- else if( _direction == Direction.Row && mc.rowsKnown() )
- ret = new long[]{mc.getRows(), 1, -1};
-
- return ret;
- }
-
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //forced / memory-based / threshold-based decision
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = findExecTypeByMemEstimate();
- }
- // Choose CP, if the input dimensions are below threshold or if the input is a vector
- else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //spark-specific decision refinement (execute unary aggregate w/ spark input and
- //single parent also in spark because it's likely cheap and reduces data transfer)
- if( _etype == ExecType.CP && _etypeForced != ExecType.CP
- && !(getInput().get(0) instanceof DataOp) //input is not checkpoint
- && getInput().get(0).getParent().size()==1 //uagg is only parent
- && getInput().get(0).optFindExecType() == ExecType.SPARK )
- {
- //pull unary aggregate into spark
- _etype = ExecType.SPARK;
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
-
- return _etype;
- }
-
- /**
- *
- * @param input
- * @param dir
- * @return
- */
- private boolean requiresAggregation( Hop input, Direction dir )
- {
- if( !ALLOW_UNARYAGG_WO_FINAL_AGG )
- return false; //customization not allowed
-
- boolean noAggRequired =
- ( input.getDim1()>1 && input.getDim1()<=input.getRowsInBlock() && dir==Direction.Col ) //e.g., colSums(X) with nrow(X)<=1000
- ||( input.getDim2()>1 && input.getDim2()<=input.getColsInBlock() && dir==Direction.Row ); //e.g., rowSums(X) with ncol(X)<=1000
-
- return !noAggRequired;
- }
-
-
- /**
- *
- * @param agg
- * @return
- */
- private SparkAggType getSparkUnaryAggregationType( boolean agg )
- {
- if( !agg )
- return SparkAggType.NONE;
-
- if( getDataType()==DataType.SCALAR //in case of scalars the block dims are not set
- || dimsKnown() && getDim1()<=getRowsInBlock() && getDim2()<=getColsInBlock() )
- return SparkAggType.SINGLE_BLOCK;
- else
- return SparkAggType.MULTI_BLOCK;
- }
-
- /**
- *
- * @return
- */
- private boolean isTernaryAggregateRewriteApplicable()
- {
- boolean ret = false;
-
- //currently we support only sum over binary multiply but potentially
- //it can be generalized to any RC aggregate over two common binary operations
- if( _direction == Direction.RowCol
- && _op == AggOp.SUM )
- {
- Hop input1 = getInput().get(0);
- if( input1 instanceof BinaryOp && ((BinaryOp)input1).getOp()==OpOp2.MULT )
- {
- Hop input11 = input1.getInput().get(0);
- Hop input12 = input1.getInput().get(1);
-
- if( input11 instanceof BinaryOp && ((BinaryOp)input11).getOp()==OpOp2.MULT )
- {
- //ternary, arbitrary matrices but no mv/outer operations.
- ret = HopRewriteUtils.isEqualSize(input11.getInput().get(0), input1)
- && HopRewriteUtils.isEqualSize(input11.getInput().get(1), input1)
- && HopRewriteUtils.isEqualSize(input12, input1);
- }
- else if( input12 instanceof BinaryOp && ((BinaryOp)input12).getOp()==OpOp2.MULT )
- {
- //ternary, arbitrary matrices but no mv/outer operations.
- ret = HopRewriteUtils.isEqualSize(input12.getInput().get(0), input1)
- && HopRewriteUtils.isEqualSize(input12.getInput().get(1), input1)
- && HopRewriteUtils.isEqualSize(input11, input1);
- }
- else
- {
- //binary, arbitrary matrices but no mv/outer operations.
- ret = HopRewriteUtils.isEqualSize(input11, input12);
- }
- }
- }
-
- return ret;
- }
-
- private static boolean isCompareOperator(OpOp2 opOp2)
- {
- return (opOp2 == OpOp2.LESS || opOp2 == OpOp2.LESSEQUAL
- || opOp2 == OpOp2.GREATER || opOp2 == OpOp2.GREATEREQUAL
- || opOp2 == OpOp2.EQUAL || opOp2 == OpOp2.NOTEQUAL);
- }
-
- /**
- *
- * @return
- */
- private boolean isUnaryAggregateOuterRewriteApplicable()
- {
- boolean ret = false;
- Hop input = getInput().get(0);
-
- if( input instanceof BinaryOp && ((BinaryOp)input).isOuterVectorOperator() )
- {
- //for special cases, we need to hold the broadcast twice in order to allow for
- //an efficient binary search over a plain java array
- double factor = (isCompareOperator(((BinaryOp)input).getOp())
- && (_direction == Direction.Row || _direction == Direction.Col || _direction == Direction.RowCol)
- && (_op == AggOp.SUM)) ? 2.0 : 1.0;
-
- factor += (isCompareOperator(((BinaryOp)input).getOp())
- && (_direction == Direction.Row || _direction == Direction.Col)
- && (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX))
- ? 1.0 : 0.0;
-
- //note: memory constraint only needs to take the rhs into account because the output
- //is guaranteed to be an aggregate of <=16KB
- Hop right = input.getInput().get(1);
- if( (right.dimsKnown() && factor*OptimizerUtils.estimateSize(right.getDim1(), right.getDim2())
- < OptimizerUtils.getRemoteMemBudgetMap(true)) //dims known and estimate fits
- ||(!right.dimsKnown() && factor*right.getOutputMemEstimate()
- < OptimizerUtils.getRemoteMemBudgetMap(true)))//dims unknown but worst-case estimate fits
- {
- ret = true;
- }
- }
-
- return ret;
- }
-
- /**
- * This will check if there is sufficient memory locally (twice the size of second matrix, for original and sort data), and remotely (size of second matrix (sorted data)).
- * @return
- */
- private boolean isUnaryAggregateOuterSPRewriteApplicable()
- {
- boolean ret = false;
- Hop input = getInput().get(0);
-
- if( input instanceof BinaryOp && ((BinaryOp)input).isOuterVectorOperator() )
- {
- //note: both cases (partitioned matrix, and sorted double array), require to
- //fit the broadcast twice into the local memory budget. Also, the memory
- //constraint only needs to take the rhs into account because the output is
- //guaranteed to be an aggregate of <=16KB
-
- Hop right = input.getInput().get(1);
-
- double size = right.dimsKnown() ?
- OptimizerUtils.estimateSize(right.getDim1(), right.getDim2()) : //dims known and estimate fits
- right.getOutputMemEstimate(); //dims unknown but worst-case estimate fits
-
- if(_op == AggOp.MAXINDEX || _op == AggOp.MININDEX){
- double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
- double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
-
- //basic requirement: the broadcast needs to to fit twice in the remote broadcast memory
- //and local memory budget because we have to create a partitioned broadcast
- //memory and hand it over to the spark context as in-memory object
- ret = ( 2*size < memBudgetExec && 2*size < memBudgetLocal );
-
- } else {
- if( OptimizerUtils.checkSparkBroadcastMemoryBudget(size) ) {
- ret = true;
- }
- }
-
- }
-
- return ret;
- }
-
-
-
- /**
- * This will check if this is one of the operator from supported LibMatrixOuterAgg library.
- * It needs to be Outer, aggregator type SUM, RowIndexMin, RowIndexMax and 6 operators <, <=, >, >=, == and !=
- *
- *
- * @return
- */
- private boolean isUnaryAggregateOuterCPRewriteApplicable()
- {
- boolean ret = false;
- Hop input = getInput().get(0);
-
- if(( input instanceof BinaryOp && ((BinaryOp)input).isOuterVectorOperator() )
- && (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX || _op == AggOp.SUM)
- && (isCompareOperator(((BinaryOp)input).getOp())))
- ret = true;
-
- return ret;
- }
-
-
- /**
- *
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- private Lop constructLopsTernaryAggregateRewrite(ExecType et)
- throws HopsException, LopsException
- {
- Hop input1 = getInput().get(0);
- Hop input11 = input1.getInput().get(0);
- Hop input12 = input1.getInput().get(1);
-
- Lop ret = null;
- Lop in1 = null;
- Lop in2 = null;
- Lop in3 = null;
-
- if( input11 instanceof BinaryOp && ((BinaryOp)input11).getOp()==OpOp2.MULT )
- {
- in1 = input11.getInput().get(0).constructLops();
- in2 = input11.getInput().get(1).constructLops();
- in3 = input12.constructLops();
- }
- else if( input12 instanceof BinaryOp && ((BinaryOp)input12).getOp()==OpOp2.MULT )
- {
- in1 = input11.constructLops();
- in2 = input12.getInput().get(0).constructLops();
- in3 = input12.getInput().get(1).constructLops();
- }
- else
- {
- in1 = input11.constructLops();
- in2 = input12.constructLops();
- in3 = new LiteralOp(1).constructLops();
- }
-
- //create new ternary aggregate operator
- int k = OptimizerUtils.getConstrainedNumThreads( _maxNumThreads );
- ret = new TernaryAggregate(in1, in2, in3, Aggregate.OperationTypes.KahanSum,
- Binary.OperationTypes.MULTIPLY, DataType.SCALAR, ValueType.DOUBLE, et, k);
-
- return ret;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- if (getDataType() != DataType.SCALAR)
- {
- Hop input = getInput().get(0);
- if ( _direction == Direction.Col ) //colwise computations
- {
- setDim1(1);
- setDim2(input.getDim2());
- }
- else if ( _direction == Direction.Row )
- {
- setDim1(input.getDim1());
- setDim2(1);
- }
- }
- }
-
- @Override
- public boolean isTransposeSafe()
- {
- boolean ret = (_direction == Direction.RowCol) && //full aggregate
- (_op == AggOp.SUM || _op == AggOp.SUM_SQ || //valid aggregration functions
- _op == AggOp.MIN || _op == AggOp.MAX ||
- _op == AggOp.PROD || _op == AggOp.MEAN);
- //note: trace and maxindex are not transpose-safe.
-
- return ret;
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- AggUnaryOp ret = new AggUnaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._direction = _direction;
- ret._maxNumThreads = _maxNumThreads;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof AggUnaryOp) )
- return false;
-
- AggUnaryOp that2 = (AggUnaryOp)that;
- return ( _op == that2._op
- && _direction == that2._direction
- && _maxNumThreads == that2._maxNumThreads
- && getInput().get(0) == that2.getInput().get(0));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/BinaryOp.java b/src/main/java/com/ibm/bi/dml/hops/BinaryOp.java
deleted file mode 100644
index 9d6d6fb..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/BinaryOp.java
+++ /dev/null
@@ -1,1565 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.lops.Aggregate;
-import com.ibm.bi.dml.lops.AppendGAlignedSP;
-import com.ibm.bi.dml.lops.AppendM;
-import com.ibm.bi.dml.lops.AppendCP;
-import com.ibm.bi.dml.lops.AppendG;
-import com.ibm.bi.dml.lops.AppendR;
-import com.ibm.bi.dml.lops.Binary;
-import com.ibm.bi.dml.lops.BinaryScalar;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.BinaryUAggChain;
-import com.ibm.bi.dml.lops.CentralMoment;
-import com.ibm.bi.dml.lops.CoVariance;
-import com.ibm.bi.dml.lops.CombineBinary;
-import com.ibm.bi.dml.lops.CombineUnary;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.DataPartition;
-import com.ibm.bi.dml.lops.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.PartialAggregate;
-import com.ibm.bi.dml.lops.PickByCount;
-import com.ibm.bi.dml.lops.RepMat;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.lops.Unary;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.CombineBinary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.mapred.DistributedCacheInput;
-
-
-/* Binary (cell operations): aij + bij
- * Properties:
- * Symbol: *, -, +, ...
- * 2 Operands
- * Semantic: align indices (sort), then perform operation
- */
-
-public class BinaryOp extends Hop
-{
-
- //we use the full remote memory budget (but reduced by sort buffer),
- public static final double APPEND_MEM_MULTIPLIER = 1.0;
-
- private Hop.OpOp2 op;
- private boolean outer = false;
-
- public static AppendMethod FORCED_APPEND_METHOD = null;
- public enum AppendMethod {
- CP_APPEND, //in-memory general case append
- MR_MAPPEND, //map-only append (rhs must be vector and fit in mapper mem)
- MR_RAPPEND, //reduce-only append (output must have at most one column block)
- MR_GAPPEND, //map-reduce general case append (map-extend, aggregate)
- SP_GAlignedAppend // special case for general case in Spark where left.getCols() % left.getColsPerBlock() == 0
- };
-
- private enum MMBinaryMethod{
- CP_BINARY,
- MR_BINARY_R, //both mm, mv
- MR_BINARY_M, //only mv (mr/spark)
- MR_BINARY_OUTER_M,
- MR_BINARY_OUTER_R, //only vv
- MR_BINARY_UAGG_CHAIN, //(mr/spark)
- }
-
- private BinaryOp() {
- //default constructor for clone
- }
-
- public BinaryOp(String l, DataType dt, ValueType vt, Hop.OpOp2 o,
- Hop inp1, Hop inp2) {
- super(l, dt, vt);
- op = o;
- getInput().add(0, inp1);
- getInput().add(1, inp2);
-
- inp1.getParent().add(this);
- inp2.getParent().add(this);
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- public OpOp2 getOp() {
- return op;
- }
-
- public void setOp(OpOp2 iop) {
- op = iop;
- }
-
- public void setOuterVectorOperation(boolean flag) {
- outer = flag;
- }
-
- public boolean isOuterVectorOperator(){
- return outer;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- //select the execution type
- ExecType et = optFindExecType();
-
- switch(op)
- {
- case IQM: {
- constructLopsIQM(et);
- break;
- }
- case CENTRALMOMENT: {
- constructLopsCentralMoment(et);
- break;
- }
- case COVARIANCE: {
- constructLopsCovariance(et);
- break;
- }
- case QUANTILE:
- case INTERQUANTILE: {
- constructLopsQuantile(et);
- break;
- }
- case MEDIAN: {
- constructLopsMedian(et);
- break;
- }
- case CBIND:
- case RBIND: {
- constructLopsAppend(et);
- break;
- }
- default:
- constructLopsBinaryDefault();
- }
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- private void constructLopsIQM(ExecType et) throws HopsException, LopsException {
- if ( et == ExecType.MR ) {
- CombineBinary combine = CombineBinary.constructCombineLop(
- OperationTypes.PreSort, (Lop) getInput().get(0)
- .constructLops(), (Lop) getInput().get(1)
- .constructLops(), DataType.MATRIX,
- getValueType());
- combine.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- SortKeys sort = SortKeys.constructSortByValueLop(
- combine,
- SortKeys.OperationTypes.WithWeights,
- DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
-
- // Sort dimensions are same as the first input
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- Data lit = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.25));
- setLineNumbers(lit);
-
- PickByCount pick = new PickByCount(
- sort, lit, DataType.MATRIX, getValueType(),
- PickByCount.OperationTypes.RANGEPICK);
-
- pick.getOutputParameters().setDimensions(-1, -1,
- getRowsInBlock(), getColsInBlock(), -1);
- setLineNumbers(pick);
-
- PartialAggregate pagg = new PartialAggregate(pick,
- HopsAgg2Lops.get(Hop.AggOp.SUM),
- HopsDirection2Lops.get(Hop.Direction.RowCol),
- DataType.MATRIX, getValueType());
- setLineNumbers(pagg);
-
- // Set the dimensions of PartialAggregate LOP based on the
- // direction in which aggregation is performed
- pagg.setDimensionsBasedOnDirection(getDim1(), getDim2(),
- getRowsInBlock(), getColsInBlock());
-
- Group group1 = new Group(pagg, Group.OperationTypes.Sort,
- DataType.MATRIX, getValueType());
- setOutputDimensions(group1);
- setLineNumbers(group1);
-
- Aggregate agg1 = new Aggregate(group1, HopsAgg2Lops
- .get(Hop.AggOp.SUM), DataType.MATRIX,
- getValueType(), ExecType.MR);
- setOutputDimensions(agg1);
- agg1.setupCorrectionLocation(pagg.getCorrectionLocation());
- setLineNumbers(agg1);
-
- UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS
- .get(OpOp1.CAST_AS_SCALAR), DataType.SCALAR,
- getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
-
- Unary iqm = new Unary(sort, unary1, Unary.OperationTypes.MR_IQM, DataType.SCALAR, ValueType.DOUBLE, ExecType.CP);
- iqm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(iqm);
- setLops(iqm);
- }
- else
- {
- SortKeys sort = SortKeys.constructSortByValueLop(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- SortKeys.OperationTypes.WithWeights,
- getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
- PickByCount pick = new PickByCount(
- sort,
- null,
- getDataType(),
- getValueType(),
- PickByCount.OperationTypes.IQM, et, true);
-
- setOutputDimensions(pick);
- setLineNumbers(pick);
- setLops(pick);
- }
- }
-
- private void constructLopsMedian(ExecType et) throws HopsException, LopsException {
- if ( et == ExecType.MR ) {
- CombineBinary combine = CombineBinary
- .constructCombineLop(
- OperationTypes.PreSort,
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- DataType.MATRIX, getValueType());
-
- SortKeys sort = SortKeys
- .constructSortByValueLop(
- combine,
- SortKeys.OperationTypes.WithWeights,
- DataType.MATRIX, getValueType(), et);
-
- combine.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- // Sort dimensions are same as the first input
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- ExecType et_pick = ExecType.CP;
-
- PickByCount pick = new PickByCount(
- sort,
- Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)),
- getDataType(),
- getValueType(),
- PickByCount.OperationTypes.MEDIAN, et_pick, false);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- setLops(pick);
- }
- else {
- SortKeys sort = SortKeys.constructSortByValueLop(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- SortKeys.OperationTypes.WithWeights,
- getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
- PickByCount pick = new PickByCount(
- sort,
- Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)),
- getDataType(),
- getValueType(),
- PickByCount.OperationTypes.MEDIAN, et, true);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- setLops(pick);
- }
- }
-
- private void constructLopsCentralMoment(ExecType et)
- throws HopsException, LopsException
- {
- // The output data type is a SCALAR if central moment
- // gets computed in CP/SPARK, and it will be MATRIX otherwise.
- DataType dt = (et == ExecType.MR ? DataType.MATRIX : DataType.SCALAR );
- CentralMoment cm = new CentralMoment(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- dt, getValueType(), et);
-
- setLineNumbers(cm);
-
- if ( et == ExecType.MR ) {
- cm.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
- UnaryCP unary1 = new UnaryCP(cm, HopsOpOp1LopsUS
- .get(OpOp1.CAST_AS_SCALAR), getDataType(),
- getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else {
- cm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLops(cm);
- }
- }
-
- /**
- *
- * @param et
- * @throws LopsException
- * @throws HopsException
- */
- private void constructLopsCovariance(ExecType et)
- throws LopsException, HopsException
- {
- if ( et == ExecType.MR ) {
- // combineBinary -> CoVariance -> CastAsScalar
- CombineBinary combine = CombineBinary.constructCombineLop(
- OperationTypes.PreCovUnweighted, getInput().get(
- 0).constructLops(), getInput().get(1)
- .constructLops(), DataType.MATRIX,
- getValueType());
-
- combine.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- CoVariance cov = new CoVariance(combine, DataType.MATRIX,
- getValueType(), et);
- cov.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
- setLineNumbers(cov);
-
- UnaryCP unary1 = new UnaryCP(cov, HopsOpOp1LopsUS
- .get(OpOp1.CAST_AS_SCALAR), getDataType(),
- getValueType());
- unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(unary1);
- setLops(unary1);
- }
- else //CP/SPARK
- {
- CoVariance cov = new CoVariance(
- getInput().get(0).constructLops(),
- getInput().get(1).constructLops(),
- getDataType(), getValueType(), et);
- cov.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(cov);
- setLops(cov);
- }
- }
-
- private void constructLopsQuantile(ExecType et) throws HopsException, LopsException {
- // 1st arguments needs to be a 1-dimensional matrix
- // For QUANTILE: 2nd argument is scalar or 1-dimensional matrix
- // For INTERQUANTILE: 2nd argument is always a scalar
-
- PickByCount.OperationTypes pick_op = null;
- if(op == Hop.OpOp2.QUANTILE)
- pick_op = PickByCount.OperationTypes.VALUEPICK;
- else
- pick_op = PickByCount.OperationTypes.RANGEPICK;
-
- if ( et == ExecType.MR )
- {
- CombineUnary combine = CombineUnary.constructCombineLop(
- getInput().get(0).constructLops(),
- getDataType(), getValueType());
-
- SortKeys sort = SortKeys.constructSortByValueLop(
- combine, SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, et);
-
- combine.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- // Sort dimensions are same as the first input
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
-
- // If only a single quantile is computed, then "pick" operation executes in CP.
- ExecType et_pick = (getInput().get(1).getDataType() == DataType.SCALAR ? ExecType.CP : ExecType.MR);
-
- PickByCount pick = new PickByCount(
- sort,
- getInput().get(1).constructLops(),
- getDataType(),
- getValueType(),
- pick_op, et_pick, false);
-
- pick.getOutputParameters().setDimensions(getDim1(),
- getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
-
- pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- setLops(pick);
- }
- else //CP/SPARK
- {
- SortKeys sort = SortKeys.constructSortByValueLop(
- getInput().get(0).constructLops(),
- SortKeys.OperationTypes.WithoutWeights,
- DataType.MATRIX, ValueType.DOUBLE, et );
- sort.getOutputParameters().setDimensions(
- getInput().get(0).getDim1(),
- getInput().get(0).getDim2(),
- getInput().get(0).getRowsInBlock(),
- getInput().get(0).getColsInBlock(),
- getInput().get(0).getNnz());
- PickByCount pick = new PickByCount( sort, getInput().get(1).constructLops(),
- getDataType(), getValueType(), pick_op, et, true);
-
- setOutputDimensions(pick);
- setLineNumbers(pick);
- setLops(pick);
- }
- }
-
- /**
- *
- * @param et
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsAppend(ExecType et)
- throws HopsException, LopsException
- {
- DataType dt1 = getInput().get(0).getDataType();
- DataType dt2 = getInput().get(1).getDataType();
- ValueType vt1 = getInput().get(0).getValueType();
- ValueType vt2 = getInput().get(1).getValueType();
- boolean cbind = op==OpOp2.CBIND;
-
- //sanity check for input data types
- if( !((dt1==DataType.MATRIX && dt2==DataType.MATRIX)
- ||(dt1==DataType.SCALAR && dt2==DataType.SCALAR
- && vt1==ValueType.STRING && vt2==ValueType.STRING )) )
- {
- throw new HopsException("Append can only apply to two matrices or two scalar strings!");
- }
-
- Lop append = null;
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX )
- {
- long rlen = cbind ? getInput().get(0).getDim1() : (getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ?
- getInput().get(0).getDim1()+getInput().get(1).getDim1() : -1;
- long clen = cbind ? ((getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ?
- getInput().get(0).getDim2()+getInput().get(1).getDim2() : -1) : getInput().get(0).getDim2();
-
- if( et == ExecType.MR )
- {
- append = constructMRAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
- }
- else if(et == ExecType.SPARK)
- {
- append = constructSPAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
- append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
- }
- else //CP
- {
- Lop offset = createOffsetLop( getInput().get(0), cbind ); //offset 1st input
- append = new AppendCP(getInput().get(0).constructLops(), getInput().get(1).constructLops(), offset, getDataType(), getValueType(), cbind);
- append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
- }
- }
- else //SCALAR-STRING and SCALAR-STRING (always CP)
- {
- append = new AppendCP(getInput().get(0).constructLops(), getInput().get(1).constructLops(),
- Data.createLiteralLop(ValueType.INT, "-1"), getDataType(), getValueType(), cbind);
- append.getOutputParameters().setDimensions(0,0,-1,-1,-1);
- }
-
- setLineNumbers(append);
- setLops(append);
- }
-
- /**
- *
- * @throws HopsException
- * @throws LopsException
- */
- private void constructLopsBinaryDefault()
- throws HopsException, LopsException
- {
- /* Default behavior for BinaryOp */
- // it depends on input data types
- DataType dt1 = getInput().get(0).getDataType();
- DataType dt2 = getInput().get(1).getDataType();
-
- if (dt1 == dt2 && dt1 == DataType.SCALAR) {
-
- // Both operands scalar
- BinaryScalar binScalar1 = new BinaryScalar(getInput().get(0)
- .constructLops(),
- getInput().get(1).constructLops(), HopsOpOp2LopsBS
- .get(op), getDataType(), getValueType());
- binScalar1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
- setLineNumbers(binScalar1);
- setLops(binScalar1);
-
- }
- else if ((dt1 == DataType.MATRIX && dt2 == DataType.SCALAR)
- || (dt1 == DataType.SCALAR && dt2 == DataType.MATRIX)) {
-
- // One operand is Matrix and the other is scalar
- ExecType et = optFindExecType();
-
- //select specific operator implementations
- Unary.OperationTypes ot = null;
- Hop right = getInput().get(1);
- if( op==OpOp2.POW && right instanceof LiteralOp && ((LiteralOp)right).getDoubleValue()==2.0 )
- ot = Unary.OperationTypes.POW2;
- else if( op==OpOp2.MULT && right instanceof LiteralOp && ((LiteralOp)right).getDoubleValue()==2.0 )
- ot = Unary.OperationTypes.MULTIPLY2;
- else //general case
- ot = HopsOpOp2LopsU.get(op);
-
-
- Unary unary1 = new Unary(getInput().get(0).constructLops(),
- getInput().get(1).constructLops(), ot, getDataType(), getValueType(), et);
-
- setOutputDimensions(unary1);
- setLineNumbers(unary1);
- setLops(unary1);
-
- }
- else
- {
- // Both operands are Matrixes
- ExecType et = optFindExecType();
- if ( et == ExecType.CP )
- {
- Binary binary = new Binary(getInput().get(0).constructLops(), getInput().get(1).constructLops(), HopsOpOp2LopsB.get(op),
- getDataType(), getValueType(), et);
-
- setOutputDimensions(binary);
- setLineNumbers(binary);
- setLops(binary);
- }
- else if(et == ExecType.SPARK)
- {
- Hop left = getInput().get(0);
- Hop right = getInput().get(1);
- MMBinaryMethod mbin = optFindMMBinaryMethodSpark(left, right);
-
- Lop binary = null;
- if( mbin == MMBinaryMethod.MR_BINARY_UAGG_CHAIN ) {
- AggUnaryOp uRight = (AggUnaryOp)right;
- binary = new BinaryUAggChain(left.constructLops(), HopsOpOp2LopsB.get(op),
- HopsAgg2Lops.get(uRight.getOp()), HopsDirection2Lops.get(uRight.getDirection()),
- getDataType(), getValueType(), et);
- }
- else if (mbin == MMBinaryMethod.MR_BINARY_M) {
- boolean partitioned = false;
- boolean isColVector = (right.getDim2()==1 && left.getDim1()==right.getDim1());
-
- binary = new BinaryM(left.constructLops(), right.constructLops(),
- HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et, partitioned, isColVector);
- }
- else {
- binary = new Binary(left.constructLops(), right.constructLops(),
- HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
- }
-
- setOutputDimensions(binary);
- setLineNumbers(binary);
- setLops(binary);
- }
- else //MR
- {
- Hop left = getInput().get(0);
- Hop right = getInput().get(1);
- MMBinaryMethod mbin = optFindMMBinaryMethod(left, right);
-
- if( mbin == MMBinaryMethod.MR_BINARY_M )
- {
- boolean needPart = requiresPartitioning(right);
- Lop dcInput = right.constructLops();
- if( needPart ) {
- //right side in distributed cache
- ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz()))
- < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : ExecType.MR; //operator selection
- dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, (right.getDim2()==1)?PDataPartitionFormat.ROW_BLOCK_WISE_N:PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
- dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
- dcInput.setAllPositions(right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
- }
-
- BinaryM binary = new BinaryM(left.constructLops(), dcInput, HopsOpOp2LopsB.get(op),
- getDataType(), getValueType(), ExecType.MR, needPart, (right.getDim2()==1 && left.getDim1()==right.getDim1()));
- setOutputDimensions(binary);
- setLineNumbers(binary);
- setLops(binary);
- }
- else if( mbin == MMBinaryMethod.MR_BINARY_UAGG_CHAIN )
- {
- AggUnaryOp uRight = (AggUnaryOp)right;
- BinaryUAggChain bin = new BinaryUAggChain(left.constructLops(), HopsOpOp2LopsB.get(op),
- HopsAgg2Lops.get(uRight.getOp()), HopsDirection2Lops.get(uRight.getDirection()),
- getDataType(), getValueType(), et);
- setOutputDimensions(bin);
- setLineNumbers(bin);
- setLops(bin);
- }
- else if( mbin == MMBinaryMethod.MR_BINARY_OUTER_R )
- {
- boolean requiresRepLeft = (!right.dimsKnown() || right.getDim2() > right.getColsInBlock());
- boolean requiresRepRight = (!left.dimsKnown() || left.getDim1() > right.getRowsInBlock());
-
- Lop leftLop = left.constructLops();
- Lop rightLop = right.constructLops();
-
- if( requiresRepLeft ) {
- Lop offset = createOffsetLop(right, true); //ncol of right determines rep of left
- leftLop = new RepMat(leftLop, offset, true, left.getDataType(), left.getValueType());
- setOutputDimensions(leftLop);
- setLineNumbers(leftLop);
- }
-
- if( requiresRepRight ) {
- Lop offset = createOffsetLop(left, false); //nrow of right determines rep of right
- rightLop = new RepMat(rightLop, offset, false, right.getDataType(), right.getValueType());
- setOutputDimensions(rightLop);
- setLineNumbers(rightLop);
- }
-
- Group group1 = new Group( leftLop, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group1);
- setOutputDimensions(group1);
-
- Group group2 = new Group( rightLop, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group2);
- setOutputDimensions(group2);
-
- Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
- setOutputDimensions(binary);
- setLineNumbers(binary);
-
- setLops(binary);
- }
- else //MMBinaryMethod.MR_BINARY_R
- {
- boolean requiresRep = requiresReplication(left, right);
-
- Lop rightLop = right.constructLops();
- if( requiresRep ) {
- Lop offset = createOffsetLop(left, (right.getDim2()<=1)); //ncol of left input (determines num replicates)
- rightLop = new RepMat(rightLop, offset, (right.getDim2()<=1), right.getDataType(), right.getValueType());
- setOutputDimensions(rightLop);
- setLineNumbers(rightLop);
- }
-
- Group group1 = new Group(getInput().get(0).constructLops(), Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group1);
- setOutputDimensions(group1);
-
- Group group2 = new Group( rightLop, Group.OperationTypes.Sort, getDataType(), getValueType());
- setLineNumbers(group2);
- setOutputDimensions(group2);
-
- Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
- setLineNumbers(binary);
- setOutputDimensions(binary);
-
- setLops(binary);
- }
- }
- }
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += "b(" + HopsOpOp2String.get(op) + ")";
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" Operation: " + op );
- for (Hop h : getInput()) {
- h.printMe();
- }
- ;
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- //preprocessing step (recognize unknowns)
- if( dimsKnown() && _nnz<0 ) //never after inference
- nnz = -1;
-
- if((op==OpOp2.CBIND || op==OpOp2.RBIND) && !OptimizerUtils.ALLOW_DYN_RECOMPILATION && !(getDataType()==DataType.SCALAR) ) {
- ret = OptimizerUtils.DEFAULT_SIZE;
- }
- else
- {
- double sparsity = 1.0;
- if( nnz < 0 ){ //check for exactly known nnz
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- if( input1.dimsKnown() && input2.dimsKnown() )
- {
- if( OptimizerUtils.isBinaryOpConditionalSparseSafe(op) && input2 instanceof LiteralOp ) {
- double sp1 = (input1.getNnz()>0 && input1.getDataType()==DataType.MATRIX) ? OptimizerUtils.getSparsity(input1.getDim1(), input1.getDim2(), input1.getNnz()) : 1.0;
- LiteralOp lit = (LiteralOp)input2;
- sparsity = OptimizerUtils.getBinaryOpSparsityConditionalSparseSafe(sp1, op, lit);
- }
- else {
- double sp1 = (input1.getNnz()>0 && input1.getDataType()==DataType.MATRIX) ? OptimizerUtils.getSparsity(input1.getDim1(), input1.getDim2(), input1.getNnz()) : 1.0;
- double sp2 = (input2.getNnz()>0 && input2.getDataType()==DataType.MATRIX) ? OptimizerUtils.getSparsity(input2.getDim1(), input2.getDim2(), input2.getNnz()) : 1.0;
- //sparsity estimates are conservative in terms of the worstcase behavior, however,
- //for outer vector operations the average case is equivalent to the worst case.
- sparsity = OptimizerUtils.getBinaryOpSparsity(sp1, sp2, op, !outer);
- }
- }
- }
- else //e.g., for append,pow or after inference
- sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
-
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
- if ( op == OpOp2.QUANTILE || op == OpOp2.IQM || op == OpOp2.MEDIAN ) {
- // buffer (=2*input_size) and output (=input_size) for SORT operation
- // getMemEstimate works for both cases of known dims and worst-case
- ret = getInput().get(0).getMemEstimate() * 3;
- }
- else if ( op == OpOp2.SOLVE ) {
- // x=solve(A,b) relies on QR decomposition of A, which is done using Apache commons-math
- // matrix of size same as the first input
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
- return interOutput;
-
- }
-
- return ret;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- MatrixCharacteristics[] mc = memo.getAllInputStats(getInput());
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- DataType dt1 = input1.getDataType();
- DataType dt2 = input2.getDataType();
-
- if( op== OpOp2.CBIND ) {
- long ldim1 = -1, ldim2 = -1, lnnz = -1;
-
- if( mc[0].rowsKnown() || mc[1].rowsKnown() )
- ldim1 = mc[0].rowsKnown() ? mc[0].getRows() : mc[1].getRows();
- if( mc[0].colsKnown() && mc[1].colsKnown() )
- ldim2 = mc[0].getCols()+mc[1].getCols();
- if( mc[0].nnzKnown() && mc[1].nnzKnown() )
- lnnz = mc[0].getNonZeros() + mc[1].getNonZeros();
-
- if( ldim1 > 0 || ldim2 > 0 || lnnz >= 0 )
- return new long[]{ldim1, ldim2, lnnz};
- }
- else if( op== OpOp2.CBIND ) {
- long ldim1 = -1, ldim2 = -1, lnnz = -1;
-
- if( mc[0].colsKnown() || mc[1].colsKnown() )
- ldim2 = mc[0].colsKnown() ? mc[0].getCols() : mc[1].getCols();
- if( mc[0].rowsKnown() && mc[1].rowsKnown() )
- ldim1 = mc[0].getRows()+mc[1].getRows();
- if( mc[0].nnzKnown() && mc[1].nnzKnown() )
- lnnz = mc[0].getNonZeros() + mc[1].getNonZeros();
-
- if( ldim1 > 0 || ldim2 > 0 || lnnz >= 0 )
- return new long[]{ldim1, ldim2, lnnz};
- }
- else if ( op == OpOp2.SOLVE ) {
- // Output is a (likely to be dense) vector of size number of columns in the first input
- if ( mc[0].getCols() > 0 ) {
- ret = new long[]{ mc[0].getCols(), 1, mc[0].getCols()};
- }
- }
- else //general case
- {
- long ldim1, ldim2;
- double sp1 = 1.0, sp2 = 1.0;
-
- if( dt1 == DataType.MATRIX && dt2 == DataType.SCALAR && mc[0].dimsKnown() )
- {
- ldim1 = mc[0].getRows();
- ldim2 = mc[0].getCols();
- sp1 = (mc[0].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[0].getNonZeros()):1.0;
- }
- else if( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX )
- {
- ldim1 = mc[1].getRows();
- ldim2 = mc[1].getCols();
- sp2 = (mc[1].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[1].getNonZeros()):1.0;
- }
- else //MATRIX - MATRIX
- {
- //propagate if either input is known, rows need always be identical,
- //for cols we need to be careful with regard to matrix-vector operations
- if( outer ) //OUTER VECTOR OPERATION
- {
- ldim1 = mc[0].getRows();
- ldim2 = mc[1].getCols();
- }
- else //GENERAL CASE
- {
- ldim1 = (mc[0].getRows()>0) ? mc[0].getRows() :
- (mc[1].getRows()>1) ? mc[1].getRows() : -1;
- ldim2 = (mc[0].getCols()>0) ? mc[0].getCols() :
- (mc[1].getCols()>1) ? mc[1].getCols() : -1;
- }
- sp1 = (mc[0].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[0].getNonZeros()):1.0;
- sp2 = (mc[1].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[1].getNonZeros()):1.0;
- }
-
- if( ldim1>0 && ldim2>0 )
- {
- if( OptimizerUtils.isBinaryOpConditionalSparseSafe(op) && input2 instanceof LiteralOp ) {
- long lnnz = (long) (ldim1*ldim2*OptimizerUtils.getBinaryOpSparsityConditionalSparseSafe(sp1, op,(LiteralOp)input2));
- ret = new long[]{ldim1, ldim2, lnnz};
- }
- else
- {
- //sparsity estimates are conservative in terms of the worstcase behavior, however,
- //for outer vector operations the average case is equivalent to the worst case.
- long lnnz = (long) (ldim1*ldim2*OptimizerUtils.getBinaryOpSparsity(sp1, sp2, op, !outer));
- ret = new long[]{ldim1, ldim2, lnnz};
- }
- }
- }
-
- return ret;
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
- DataType dt1 = getInput().get(0).getDataType();
- DataType dt2 = getInput().get(1).getDataType();
-
- if( _etypeForced != null ) {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = findExecTypeByMemEstimate();
- }
- else
- {
- _etype = null;
- if ( dt1 == DataType.MATRIX && dt2 == DataType.MATRIX ) {
- // choose CP if the dimensions of both inputs are below Hops.CPThreshold
- // OR if both are vectors
- if ( (getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold())
- || (getInput().get(0).isVector() && getInput().get(1).isVector()))
- {
- _etype = ExecType.CP;
- }
- }
- else if ( dt1 == DataType.MATRIX && dt2 == DataType.SCALAR ) {
- if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() )
- {
- _etype = ExecType.CP;
- }
- }
- else if ( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX ) {
- if ( getInput().get(1).areDimsBelowThreshold() || getInput().get(1).isVector() )
- {
- _etype = ExecType.CP;
- }
- }
- else
- {
- _etype = ExecType.CP;
- }
-
- //if no CP condition applied
- if( _etype == null )
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //spark-specific decision refinement (execute unary scalar w/ spark input and
- //single parent also in spark because it's likely cheap and reduces intermediates)
- if( _etype == ExecType.CP && _etypeForced != ExecType.CP
- && getDataType().isMatrix() && (dt1.isScalar() || dt2.isScalar())
- && supportsMatrixScalarOperations() //scalar operations
- && !(getInput().get(dt1.isScalar()?1:0) instanceof DataOp) //input is not checkpoint
- && getInput().get(dt1.isScalar()?1:0).getParent().size()==1 //unary scalar is only parent
- && !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar()?1:0)) //single block triggered exec
- && getInput().get(dt1.isScalar()?1:0).optFindExecType() == ExecType.SPARK )
- {
- //pull unary scalar operation into spark
- _etype = ExecType.SPARK;
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
-
- //ensure cp exec type for single-node operations
- if ( op == OpOp2.SOLVE ) {
- _etype = ExecType.CP;
- }
-
- return _etype;
- }
-
- /**
- * General case binary append.
- *
- * @param left
- * @param right
- * @param cbind
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- public static Lop constructMRAppendLop( Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current )
- throws HopsException, LopsException
- {
- Lop ret = null;
-
- long m1_dim1 = left.getDim1();
- long m1_dim2 = left.getDim2();
- long m2_dim1 = right.getDim1();
- long m2_dim2 = right.getDim2();
- long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1>0 && m2_dim1>0) ? (m1_dim1 + m2_dim1) : -1); //output rows
- long m3_dim2 = cbind ? ((m1_dim2>0 && m2_dim2>0) ? (m1_dim2 + m2_dim2) : -1): m1_dim2; //output cols
- long m3_nnz = (left.getNnz()>0 && right.getNnz()>0) ? (left.getNnz() + right.getNnz()) : -1; //output nnz
- long brlen = left.getRowsInBlock();
- long bclen = left.getColsInBlock();
-
- Lop offset = createOffsetLop( left, cbind ); //offset 1st input
- AppendMethod am = optFindAppendMethod(m1_dim1, m1_dim2, m2_dim1, m2_dim2, brlen, bclen, cbind);
-
- switch( am )
- {
- case MR_MAPPEND: //special case map-only append
- {
- boolean needPart = requiresPartitioning(right);
- //pre partitioning
- Lop dcInput = right.constructLops();
- if( needPart ) {
- //right side in distributed cache
- ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz()))
- < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : ExecType.MR; //operator selection
- dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
- dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
- dcInput.setAllPositions(right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
- }
-
- AppendM appM = new AppendM(left.constructLops(), dcInput, offset, dt, vt, cbind, needPart, ExecType.MR);
- appM.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
- appM.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
- ret = appM;
- break;
- }
- case MR_RAPPEND: //special case reduce append w/ one column block
- {
- //group
- Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m1_dim1, m1_dim2, brlen, bclen, left.getNnz());
- group1.setAllPositions(left.getBeginLine(), left.getBeginColumn(), left.getEndLine(), left.getEndColumn());
-
- Group group2 = new Group(right.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m2_dim1, m2_dim2, brlen, bclen, right.getNnz());
- group1.setAllPositions(right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
-
- AppendR appR = new AppendR(group1, group2, dt, vt, cbind, ExecType.MR);
- appR.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
- appR.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- ret = appR;
- break;
- }
- case MR_GAPPEND:
- {
- //general case: map expand append, reduce aggregate
- Lop offset2 = createOffsetLop( right, cbind ); //offset second input
-
- AppendG appG = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, dt, vt, cbind, ExecType.MR);
- appG.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
- appG.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- //group
- Group group1 = new Group(appG, Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
- group1.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- //aggregate
- Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, vt, ExecType.MR);
- agg1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
- agg1.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
- ret = agg1;
- break;
- }
- default:
- throw new HopsException("Invalid MR append method: "+am);
- }
-
- return ret;
- }
-
- /**
- *
- * @param left
- * @param right
- * @param dt
- * @param vt
- * @param current
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- public static Lop constructSPAppendLop( Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current )
- throws HopsException, LopsException
- {
- Lop ret = null;
-
- Lop offset = createOffsetLop( left, cbind ); //offset 1st input
- AppendMethod am = optFindAppendSPMethod(left.getDim1(), left.getDim2(), right.getDim1(), right.getDim2(),
- right.getRowsInBlock(), right.getColsInBlock(), right.getNnz(), cbind);
-
- switch( am )
- {
- case MR_MAPPEND: //special case map-only append
- {
- ret = new AppendM(left.constructLops(), right.constructLops(), offset,
- current.getDataType(), current.getValueType(), cbind, false, ExecType.SPARK);
- break;
- }
- case MR_RAPPEND: //special case reduce append w/ one column block
- {
- ret = new AppendR(left.constructLops(), right.constructLops(),
- current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
- break;
- }
- case MR_GAPPEND:
- {
- Lop offset2 = createOffsetLop( right, cbind ); //offset second input
- ret = new AppendG(left.constructLops(), right.constructLops(), offset, offset2,
- current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
- break;
- }
- case SP_GAlignedAppend:
- {
- ret = new AppendGAlignedSP(left.constructLops(), right.constructLops(), offset,
- current.getDataType(), current.getValueType(), cbind);
- break;
- }
- default:
- throw new HopsException("Invalid SP append method: "+am);
- }
-
- ret.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
-
- return ret;
- }
-
- /**
- * Special case tertiary append. Here, we also compile a MR_RAPPEND or MR_GAPPEND
- *
- * @param left
- * @param right
- * @param dt
- * @param vt
- * @param current
- * @return
- * @throws HopsException
- * @throws LopsException
- */
- public static Lop constructAppendLopChain( Hop left, Hop right1, Hop right2, DataType dt, ValueType vt, boolean cbind, Hop current )
- throws HopsException, LopsException
- {
- long m1_dim1 = left.getDim1();
- long m1_dim2 = left.getDim2();
- long m2_dim1 = right1.getDim1();
- long m2_dim2 = right1.getDim2();
- long m3_dim1 = right2.getDim1();
- long m3_dim2 = right2.getDim2();
- long m41_dim2 = (m1_dim2>0 && m2_dim2>0) ? (m1_dim2 + m2_dim2) : -1; //output cols
- long m41_nnz = (left.getNnz()>0 && right1.getNnz()>0) ?
- (left.getNnz() + right1.getNnz()) : -1; //output nnz
- long m42_dim2 = (m1_dim2>0 && m2_dim2>0 && m3_dim2>0) ? (m1_dim2 + m2_dim2 + m3_dim2) : -1; //output cols
- long m42_nnz = (left.getNnz()>0 && right1.getNnz()>0 && right2.getNnz()>0) ?
- (left.getNnz() + right1.getNnz()+ right2.getNnz()) : -1; //output nnz
- long brlen = left.getRowsInBlock();
- long bclen = left.getColsInBlock();
-
- //warn if assumption of blocksize>=3 does not hold
- if( bclen < 3 )
- throw new HopsException("MR_RAPPEND requires a blocksize of >= 3.");
-
- //case MR_RAPPEND:
- //special case reduce append w/ one column block
-
- Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m1_dim1, m1_dim2, brlen, bclen, left.getNnz());
- group1.setAllPositions(left.getBeginLine(), left.getBeginColumn(), left.getEndLine(), left.getEndColumn());
-
- Group group2 = new Group(right1.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m2_dim1, m2_dim2, brlen, bclen, right1.getNnz());
- group1.setAllPositions(right1.getBeginLine(), right1.getBeginColumn(), right1.getEndLine(), right1.getEndColumn());
-
- Group group3 = new Group(right2.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
- group1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, right2.getNnz());
- group1.setAllPositions(right2.getBeginLine(), right2.getBeginColumn(), right2.getEndLine(), right2.getEndColumn());
-
- AppendR appR1 = new AppendR(group1, group2, dt, vt, cbind, ExecType.MR);
- appR1.getOutputParameters().setDimensions(m1_dim1, m41_dim2, brlen, bclen, m41_nnz);
- appR1.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- AppendR appR2 = new AppendR(appR1, group3, dt, vt, cbind, ExecType.MR);
- appR1.getOutputParameters().setDimensions(m1_dim1, m42_dim2, brlen, bclen, m42_nnz);
- appR1.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- return appR2;
- }
-
- /**
- * Estimates the memory footprint of MapMult operation depending on which input is put into distributed cache.
- * This function is called by <code>optFindAppendMethod()</code> to decide the execution strategy, as well as by
- * piggybacking to decide the number of Map-side instructions to put into a single GMR job.
- */
- public static double footprintInMapper( long m1_dim1, long m1_dim2, long m2_dim1, long m2_dim2, long m1_rpb, long m1_cpb ) {
- double footprint = 0;
-
- // size of left input (matrix block)
- footprint += OptimizerUtils.estimateSize(Math.min(m1_dim1, m1_rpb), Math.min(m1_dim2, m1_cpb));
-
- // size of right input (vector)
- footprint += OptimizerUtils.estimateSize(m2_dim1, m2_dim2);
-
- // size of the output (only boundary block is merged)
- footprint += OptimizerUtils.estimateSize(Math.min(m1_dim1, m1_rpb), Math.min(m1_dim2+m2_dim2, m1_cpb));
-
- return footprint;
- }
-
- /**
- *
- * @param m1_dim1
- * @param m1_dim2
- * @param m2_dim1
- * @param m2_dim2
- * @return
- */
- private static AppendMethod optFindAppendMethod( long m1_dim1, long m1_dim2, long m2_dim1, long m2_dim2, long m1_rpb, long m1_cpb, boolean cbind )
- {
- if(FORCED_APPEND_METHOD != null) {
- return FORCED_APPEND_METHOD;
- }
-
- //check for best case (map-only)
- if( m2_dim1 >= 1 && m2_dim2 >= 1 //rhs dims known
- && (cbind && m2_dim2 <= m1_cpb //rhs is smaller than column block
- || !cbind && m2_dim1 <= m1_rpb) ) //rhs is smaller than row block
- {
- double footprint = BinaryOp.footprintInMapper(m1_dim1, m1_dim2, m2_dim1, m2_dim2, m1_rpb, m1_cpb);
- if ( footprint < APPEND_MEM_MULTIPLIER * OptimizerUtils.getRemoteMemBudgetMap(true) )
- return AppendMethod.MR_MAPPEND;
- }
-
- //check for in-block append (reduce-only)
- if( cbind && m1_dim2 >= 1 && m2_dim2 >= 0 //column dims known
- && m1_dim2+m2_dim2 <= m1_cpb //output has one column block
- ||!cbind && m1_dim1 >= 1 && m2_dim1 >= 0 //row dims known
- && m1_dim1+m2_dim1 <= m1_rpb ) //output has one column block
- {
- return AppendMethod.MR_RAPPEND;
- }
-
- //general case (map and reduce)
- return AppendMethod.MR_GAPPEND;
- }
-
- private static AppendMethod optFindAppendSPMethod( long m1_dim1, long m1_dim2, long m2_dim1, long m2_dim2, long m1_rpb, long m1_cpb, long m2_nnz, boolean cbind )
- {
- if(FORCED_APPEND_METHOD != null) {
- return FORCED_APPEND_METHOD;
- }
-
- //check for best case (map-only w/o shuffle)
- if( m2_dim1 >= 1 && m2_dim2 >= 1 //rhs dims known
- && (cbind && m2_dim2 <= m1_cpb //rhs is smaller than column block
- || !cbind && m2_dim1 <= m1_rpb) ) //rhs is smaller than row block
- {
- if( OptimizerUtils.checkSparkBroadcastMemoryBudget(m2_dim1, m2_dim2, m1_rpb, m1_cpb, m2_nnz) ) {
- return AppendMethod.MR_MAPPEND;
- }
- }
-
- //check for in-block append (reduce-only)
- if( cbind && m1_dim2 >= 1 && m2_dim2 >= 0 //column dims known
- && m1_dim2+m2_dim2 <= m1_cpb //output has one column block
- ||!cbind && m1_dim1 >= 1 && m2_dim1 >= 0 //row dims known
- && m1_dim1+m2_dim1 <= m1_rpb ) //output has one column block
- {
- return AppendMethod.MR_RAPPEND;
- }
-
- // if(mc1.getCols() % mc1.getColsPerBlock() == 0) {
- if( cbind && m1_dim2 % m1_cpb == 0
- || !cbind && m1_dim1 % m1_rpb == 0 )
- {
- return AppendMethod.SP_GAlignedAppend;
- }
-
- //general case (map and reduce)
- return AppendMethod.MR_GAPPEND;
- }
-
- /**
- *
- * @param rightInput
- * @return
- */
- private static boolean requiresPartitioning( Hop rightInput )
- {
- return ( rightInput.dimsKnown() //known input size
- && rightInput.getDim1()*rightInput.getDim2() > DistributedCacheInput.PARTITION_SIZE);
- }
-
- /**
- *
- * @param left
- * @param right
- * @return
- */
- private static boolean requiresReplication( Hop left, Hop right )
- {
- return (!(left.getDim2()>=1 && right.getDim2()>=1) //cols of any input unknown
- ||(left.getDim2() > 1 && right.getDim2()==1 && left.getDim2()>=left.getColsInBlock() ) //col MV and more than 1 block
- ||(left.getDim1() > 1 && right.getDim1()==1 && left.getDim1()>=left.getRowsInBlock() )); //row MV and more than 1 block
- }
-
- private MMBinaryMethod optFindMMBinaryMethodSpark(Hop left, Hop right) {
- long m1_dim1 = left.getDim1();
- long m1_dim2 = left.getDim2();
- long m2_dim1 = right.getDim1();
- long m2_dim2 = right.getDim2();
- long m1_rpb = left.getRowsInBlock();
- long m1_cpb = left.getColsInBlock();
-
- //MR_BINARY_UAGG_CHAIN only applied if result is column/row vector of MV binary operation.
- if( right instanceof AggUnaryOp && right.getInput().get(0) == left //e.g., P / rowSums(P)
- && ((((AggUnaryOp) right).getDirection() == Direction.Row && m1_dim2 > 1 && m1_dim2 <= m1_cpb ) //single column block
- || (((AggUnaryOp) right).getDirection() == Direction.Col && m1_dim1 > 1 && m1_dim1 <= m1_rpb ))) //single row block
- {
- return MMBinaryMethod.MR_BINARY_UAGG_CHAIN;
- }
-
- //MR_BINARY_M currently only applied for MV because potential partitioning job may cause additional latency for VV.
- if( m2_dim1 >= 1 && m2_dim2 >= 1 // rhs dims known
- && ((m1_dim2 >= 1 && m2_dim2 == 1) //rhs column vector
- ||(m1_dim1 >= 1 && m2_dim1 == 1 )) ) //rhs row vector
- {
- double size = OptimizerUtils.estimateSize(m2_dim1, m2_dim2);
- if( OptimizerUtils.checkSparkBroadcastMemoryBudget(size) ) {
- return MMBinaryMethod.MR_BINARY_M;
- }
- }
-
- //MR_BINARY_R as robust fallback strategy
- return MMBinaryMethod.MR_BINARY_R;
- }
-
- /**
- *
- * @param left
- * @param right
- * @return
- */
- private MMBinaryMethod optFindMMBinaryMethod(Hop left, Hop right)
- {
- long m1_dim1 = left.getDim1();
- long m1_dim2 = left.getDim2();
- long m2_dim1 = right.getDim1();
- long m2_dim2 = right.getDim2();
- long m1_rpb = left.getRowsInBlock();
- long m1_cpb = left.getColsInBlock();
-
- //MR_BINARY_OUTER only applied if outer vector operation
- if( outer ) {
- return MMBinaryMethod.MR_BINARY_OUTER_R;
- }
-
- //MR_BINARY_UAGG_CHAIN only applied if result is column/row vector of MV binary operation.
- if( right instanceof AggUnaryOp && right.getInput().get(0) == left //e.g., P / rowSums(P)
- && ((((AggUnaryOp) right).getDirection() == Direction.Row && m1_dim2 > 1 && m1_dim2 <= m1_cpb ) //single column block
- || (((AggUnaryOp) right).getDirection() == Direction.Col && m1_dim1 > 1 && m1_dim1 <= m1_rpb ))) //single row block
- {
- return MMBinaryMethod.MR_BINARY_UAGG_CHAIN;
- }
-
- //MR_BINARY_M currently only applied for MV because potential partitioning job may cause additional latency for VV.
- if( m2_dim1 >= 1 && m2_dim2 >= 1 // rhs dims known
- && ((m1_dim2 >1 && m2_dim2 == 1) //rhs column vector
- ||(m1_dim1 >1 && m2_dim1 == 1 )) ) //rhs row vector
- {
- double footprint = BinaryOp.footprintInMapper(m1_dim1, m1_dim2, m2_dim1, m2_dim2, m1_rpb, m1_cpb);
- if ( footprint < OptimizerUtils.getRemoteMemBudgetMap(true) )
- return MMBinaryMethod.MR_BINARY_M;
- }
-
- //MR_BINARY_R as robust fallback strategy
- return MMBinaryMethod.MR_BINARY_R;
- }
-
-
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = getInput().get(0);
- Hop input2 = getInput().get(1);
- DataType dt1 = input1.getDataType();
- DataType dt2 = input2.getDataType();
-
- if ( getDataType() == DataType.SCALAR )
- {
- //do nothing always known
- setDim1(0);
- setDim2(0);
- }
- else //MATRIX OUTPUT
- {
- //TODO quantile
- if( op == OpOp2.CBIND )
- {
- setDim1( (input1.getDim1()>0) ? input1.getDim1() : input2.getDim1() );
-
- //ensure both columns are known, otherwise dangerous underestimation due to +(-1)
- if( input1.getDim2()>0 && input2.getDim2()>0 )
- setDim2( input1.getDim2() + input2.getDim2() );
- //ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
- if( input1.getNnz()>0 && input2.getNnz()>0 )
- setNnz( input1.getNnz() + input2.getNnz() );
- }
- else if( op == OpOp2.RBIND )
- {
- setDim2( (input1.getDim2()>0) ? input1.getDim2() : input2.getDim2() );
-
- //ensure both rows are known, otherwise dangerous underestimation due to +(-1)
- if( input1.getDim1()>0 && input2.getDim1()>0 )
- setDim1( input1.getDim1() + input2.getDim1() );
- //ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
- if( input1.getNnz()>0 && input2.getNnz()>0 )
- setNnz( input1.getNnz() + input2.getNnz() );
- }
- else if ( op == OpOp2.SOLVE )
- {
- //normally the second input would be of equal size as the output
- //however, since we use qr internally, it also supports squared first inputs
- setDim1( input1.getDim2() );
- setDim2( input2.getDim2() );
- }
- else //general case
- {
- long ldim1, ldim2, lnnz1 = -1;
-
- if( dt1 == DataType.MATRIX && dt2 == DataType.SCALAR )
- {
- ldim1 = input1.getDim1();
- ldim2 = input1.getDim2();
- lnnz1 = input1.getNnz();
- }
- else if( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX )
- {
- ldim1 = input2.getDim1();
- ldim2 = input2.getDim2();
- }
- else //MATRIX - MATRIX
- {
- //propagate if either input is known, rows need always be identical,
- //for cols we need to be careful with regard to matrix-vector operations
- if( outer ) //OUTER VECTOR OPERATION
- {
- ldim1 = input1.getDim1();
- ldim2 = input2.getDim2();
- }
- else //GENERAL CASE
- {
- ldim1 = (input1.getDim1()>0) ? input1.getDim1()
- : ((input2.getDim1()>1)?input2.getDim1():-1);
- ldim2 = (input1.getDim2()>0) ? input1.getDim2()
- : ((input2.getDim2()>1)?input2.getDim2():-1);
- lnnz1 = input1.getNnz();
- }
- }
-
- setDim1( ldim1 );
- setDim2( ldim2 );
-
- //update nnz only if we can ensure exact results,
- //otherwise propagated via worst-case estimates
- if( op == OpOp2.POW
- || (input2 instanceof LiteralOp && OptimizerUtils.isBinaryOpConditionalSparseSafeExact(op, (LiteralOp)input2)) )
- {
- setNnz( lnnz1 );
- }
- }
- }
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- BinaryOp ret = new BinaryOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret.op = op;
- ret.outer = outer;
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof BinaryOp) )
- return false;
-
- BinaryOp that2 = (BinaryOp)that;
- return ( op == that2.op
- && outer == that2.outer
- && getInput().get(0) == that2.getInput().get(0)
- && getInput().get(1) == that2.getInput().get(1));
- }
-
- /**
- *
- * @param op
- * @return
- */
- public boolean supportsMatrixScalarOperations()
- {
- return ( op==OpOp2.PLUS ||op==OpOp2.MINUS
- ||op==OpOp2.MULT ||op==OpOp2.DIV
- ||op==OpOp2.MODULUS ||op==OpOp2.INTDIV
- ||op==OpOp2.LESS ||op==OpOp2.LESSEQUAL
- ||op==OpOp2.GREATER ||op==OpOp2.GREATEREQUAL
- ||op==OpOp2.EQUAL ||op==OpOp2.NOTEQUAL
- ||op==OpOp2.MIN ||op==OpOp2.MAX
- ||op==OpOp2.AND ||op==OpOp2.OR
- ||op==OpOp2.LOG ||op==OpOp2.POW );
- }
-
- /**
- *
- * @return
- */
- public boolean isPPredOperation()
- {
- return ( op==OpOp2.LESS ||op==OpOp2.LESSEQUAL
- ||op==OpOp2.GREATER ||op==OpOp2.GREATEREQUAL
- ||op==OpOp2.EQUAL ||op==OpOp2.NOTEQUAL);
- }
-}
[75/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GraphBuilder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GraphBuilder.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GraphBuilder.java
index 6507faa..3962707 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GraphBuilder.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GraphBuilder.java
@@ -15,31 +15,31 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfgraph;
+package org.apache.sysml.hops.globalopt.gdfgraph;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.Set;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.globalopt.Summary;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.utils.Explain;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.globalopt.Summary;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.utils.Explain;
/**
* GENERAL 'GDF GRAPH' STRUCTURE, by MB:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
index 10f4df1..55a4403 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristic.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
+package org.apache.sysml.hops.globalopt.gdfresolve;
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
+import org.apache.sysml.hops.globalopt.RewriteConfig;
public abstract class GDFMismatchHeuristic
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
index b9604f4..ef9caa9 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
+package org.apache.sysml.hops.globalopt.gdfresolve;
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
+import org.apache.sysml.hops.globalopt.RewriteConfig;
public class GDFMismatchHeuristicBlocksizeOrFirst extends GDFMismatchHeuristic
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
index d069aa8..3fb1bbd 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
+package org.apache.sysml.hops.globalopt.gdfresolve;
-import com.ibm.bi.dml.hops.globalopt.RewriteConfig;
+import org.apache.sysml.hops.globalopt.RewriteConfig;
public class GDFMismatchHeuristicFirst extends GDFMismatchHeuristic
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
index 0031a87..6bb64f0 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.hops.globalopt.gdfresolve;
+package org.apache.sysml.hops.globalopt.gdfresolve;
-import com.ibm.bi.dml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType;
+import org.apache.sysml.runtime.DMLRuntimeException;
public abstract class MismatchHeuristicFactory
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index 744c22f..e21f171 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.ipa;
+package org.apache.sysml.hops.ipa;
import java.util.ArrayList;
import java.util.HashMap;
@@ -30,52 +30,52 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.udf.lib.DeNaNWrapper;
-import com.ibm.bi.dml.udf.lib.DeNegInfinityWrapper;
-import com.ibm.bi.dml.udf.lib.DynamicReadMatrixCP;
-import com.ibm.bi.dml.udf.lib.DynamicReadMatrixRcCP;
-import com.ibm.bi.dml.udf.lib.OrderWrapper;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.FunctionOp.FunctionType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.udf.lib.DeNaNWrapper;
+import org.apache.sysml.udf.lib.DeNegInfinityWrapper;
+import org.apache.sysml.udf.lib.DynamicReadMatrixCP;
+import org.apache.sysml.udf.lib.DynamicReadMatrixRcCP;
+import org.apache.sysml.udf.lib.OrderWrapper;
/**
* This Inter Procedural Analysis (IPA) serves two major purposes:
@@ -103,7 +103,7 @@ import com.ibm.bi.dml.udf.lib.OrderWrapper;
* If ALLOW_MULTIPLE_FUNCTION_CALLS is enabled we treat multiple calls with the same sizes
* as one call and hence, propagate those statistics into the function as well.
* * Output size inference happens for DML-bodied functions that are invoked exactly once
- * and for external functions that are known in advance (see UDFs in com.ibm.bi.dml.udf).
+ * and for external functions that are known in advance (see UDFs in org.apache.sysml.udf).
* * Size propagation across DAGs requires control flow awareness:
* - Generic statement blocks: updated variables -> old stats in; new stats out
* - While/for statement blocks: updated variables -> old stats in/out if loop insensitive; otherwise unknown
@@ -129,7 +129,7 @@ public class InterProceduralAnalysis
static {
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.parser.InterProceduralAnalysis")
+ Logger.getLogger("org.apache.sysml.parser.InterProceduralAnalysis")
.setLevel((Level) Level.DEBUG);
}
}
@@ -831,14 +831,14 @@ public class InterProceduralAnalysis
MatrixObject moOut = createOutputMatrix(input.getDim1(), input.getDim2(),lnnz);
callVars.put(fop.getOutputVariableNames()[0], moOut);
}
- else if( className.equals("com.ibm.bi.dml.udf.lib.EigenWrapper") )
+ else if( className.equals("org.apache.sysml.udf.lib.EigenWrapper") )
//else if( className.equals(EigenWrapper.class.getName()) ) //string ref for build flexibility
{
Hop input = fop.getInput().get(0);
callVars.put(fop.getOutputVariableNames()[0], createOutputMatrix(input.getDim1(), 1, -1));
callVars.put(fop.getOutputVariableNames()[1], createOutputMatrix(input.getDim1(), input.getDim1(),-1));
}
- else if( className.equals("com.ibm.bi.dml.udf.lib.LinearSolverWrapperCP") )
+ else if( className.equals("org.apache.sysml.udf.lib.LinearSolverWrapperCP") )
//else if( className.equals(LinearSolverWrapperCP.class.getName()) ) //string ref for build flexibility
{
Hop input = fop.getInput().get(1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java b/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
index 9610be3..acb6e01 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
@@ -15,32 +15,32 @@
*
*/
-package com.ibm.bi.dml.hops.recompile;
+package org.apache.sysml.hops.recompile;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.utils.Statistics;
public class LiteralReplacement
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java b/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
index e6b8426..2c80fc9 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.hops.recompile;
+package org.apache.sysml.hops.recompile;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.parser.VariableSet;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
public class RecompileStatus
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index b8e894e..ac6c1b9 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.recompile;
+package org.apache.sysml.hops.recompile;
import java.io.BufferedReader;
import java.io.IOException;
@@ -31,76 +31,76 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.wink.json4j.JSONObject;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.lops.CSVReBlock;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptTreeConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Explain.ExplainType;
-import com.ibm.bi.dml.utils.JSONHelper;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.FunctionOp.FunctionType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.MemoTable;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.hops.rewrite.ProgramRewriter;
+import org.apache.sysml.lops.CSVReBlock;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.ReBlock;
+import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.opt.OptTreeConverter;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.utils.Explain;
+import org.apache.sysml.utils.Explain.ExplainType;
+import org.apache.sysml.utils.JSONHelper;
/**
* Dynamic recompilation of hop dags to runtime instructions, which includes the
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java b/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
index 3ea68c2..15a106f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.utils.Explain;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.utils.Explain;
/**
* This class allows to check hop dags for validity, e.g., parent-child linking.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteRule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteRule.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteRule.java
index 65bb5ad..e5176e7 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteRule.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteRule.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
/**
* Base class for all hop rewrites in order to enable generic
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 6da131c..6648f7a 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -15,46 +15,46 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.MemoTable;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class HopRewriteUtils
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriteStatus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriteStatus.java b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriteStatus.java
index 535f68c..0d3b707 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriteStatus.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriteStatus.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
public class ProgramRewriteStatus
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 9c383e9..74f048b 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
@@ -24,21 +24,21 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.FunctionStatement;
-import com.ibm.bi.dml.parser.FunctionStatementBlock;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatement;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
/**
* This program rewriter applies a variety of rule-based rewrites
@@ -60,7 +60,7 @@ public class ProgramRewriter
static{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.rewrite")
+ Logger.getLogger("org.apache.sysml.hops.rewrite")
.setLevel((Level) Level.DEBUG);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index 5b1111a..175d4d7 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,30 +23,30 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.QuaternaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.OpOp4;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.QuaternaryOp;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.OpOp4;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: Algebraic Simplifications. Simplifies binary expressions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 4a7b2d4..bdd8d8b 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,29 +23,29 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: Algebraic Simplifications. Simplifies binary expressions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
index bb4260e..be41de2 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.parser.Expression.DataType;
/**
* Rule: BlockSizeAndReblock. For all statement blocks, determine
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteCommonSubexpressionElimination.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteCommonSubexpressionElimination.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteCommonSubexpressionElimination.java
index 9bb12f2..8dae18f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteCommonSubexpressionElimination.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteCommonSubexpressionElimination.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
/**
* Rule: CommonSubexpressionElimination. For all statement blocks,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
index 8b1bf5b..35dd07b 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
@@ -15,34 +15,34 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.io.IOException;
import java.util.ArrayList;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
/**
* Rule: Constant Folding. For all statement blocks,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteForLoopVectorization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteForLoopVectorization.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteForLoopVectorization.java
index 49af864..d126fac 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteForLoopVectorization.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteForLoopVectorization.java
@@ -15,30 +15,30 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.ForStatement;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.AggUnaryOp;
+import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.AggOp;
+import org.apache.sysml.hops.Hop.Direction;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: Simplify program structure by pulling if or else statement body out
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
index 226812d..d0dbb07 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.IndexingOp;
+import org.apache.sysml.hops.LeftIndexingOp;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: Indexing vectorization. This rewrite rule set simplifies
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
index 5e0e54f..69287a7 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.parser.WhileStatementBlock;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.VariableSet;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: Insert checkpointing operations for caching purposes. Currently, we
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
index 71d913d..db81f13 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkPReadCheckpointing.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.OptimizerUtils;
/**
* Rule: BlockSizeAndReblock. For all statement blocks, determine
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteMatrixMultChainOptimization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteMatrixMultChainOptimization.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteMatrixMultChainOptimization.java
index c656f87..2e9b329 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteMatrixMultChainOptimization.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteMatrixMultChainOptimization.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,10 +25,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.utils.Explain;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.utils.Explain;
/**
* Rule: Determine the optimal order of execution for a chain of
@@ -47,7 +47,7 @@ public class RewriteMatrixMultChainOptimization extends HopRewriteRule
{
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("com.ibm.bi.dml.hops.rewrite.RewriteMatrixMultChainOptimization")
+ Logger.getLogger("org.apache.sysml.hops.rewrite.RewriteMatrixMultChainOptimization")
.setLevel((Level) Level.TRACE);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
index 4eb67b6..70a294b 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashSet;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.Hop.VisitStatus;
/**
* This rewrite is a custom rewrite for JMLC in order to replace all persistent reads
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveReadAfterWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveReadAfterWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveReadAfterWrite.java
index a37c54a..82f95ba 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveReadAfterWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveReadAfterWrite.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.DataExpression;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.parser.DataExpression;
/**
* Rule: RemoveReadAfterWrite. If there is a persistent read with the same filename
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
index 0587729..8e09e87 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.parser.IfStatement;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.StatementBlock;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
/**
* Rule: Simplify program structure by pulling if or else statement body out
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
index 011f67e..6df1b89 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryCasts.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Rule: RemoveUnnecessaryCasts. For all value type casts check
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
index 2920853..4300119 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
@@ -15,33 +15,33 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashSet;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.Hop.OpOp3;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
+import org.apache.sysml.hops.AggBinaryOp;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.OpOp3;
+import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.ReOrgOp;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.LiteralOp;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.hops.ReorgOp;
+import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.hops.UnaryOp;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.VariableSet;
+import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.matrix.data.Pair;
/**
* Rule: Split Hop DAG after specific data-dependent operators. This is
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
index 799f8ab..1cffee1 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.Hop.FileFormatTypes;
+import org.apache.sysml.hops.Hop.VisitStatus;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.VariableSet;
/**
* Rule: Split Hop DAG after CSV reads with unknown size. This is
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/RewriteTransientWriteParentHandling.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteTransientWriteParentHandling.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteTransientWriteParentHandling.java
index ee9c010..d9e841e 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteTransientWriteParentHandling.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteTransientWriteParentHandling.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
+import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.hops.Hop.DataOpTypes;
/**
* Rule: Eliminate for Transient Write DataHops to have no parents
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/hops/rewrite/StatementBlockRewriteRule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/StatementBlockRewriteRule.java b/src/main/java/org/apache/sysml/hops/rewrite/StatementBlockRewriteRule.java
index 0e4c7b8..9d84f2f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/StatementBlockRewriteRule.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/StatementBlockRewriteRule.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.hops.rewrite;
+package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.StatementBlock;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.StatementBlock;
/**
* Base class for all hop rewrites in order to enable generic
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Aggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Aggregate.java b/src/main/java/org/apache/sysml/lops/Aggregate.java
index 88d57e7..5149677 100644
--- a/src/main/java/org/apache/sysml/lops/Aggregate.java
+++ b/src/main/java/org/apache/sysml/lops/Aggregate.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/AppendCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/AppendCP.java b/src/main/java/org/apache/sysml/lops/AppendCP.java
index f3108e5..b98db98 100644
--- a/src/main/java/org/apache/sysml/lops/AppendCP.java
+++ b/src/main/java/org/apache/sysml/lops/AppendCP.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
public class AppendCP extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/AppendG.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/AppendG.java b/src/main/java/org/apache/sysml/lops/AppendG.java
index f254aa0..05f2a32 100644
--- a/src/main/java/org/apache/sysml/lops/AppendG.java
+++ b/src/main/java/org/apache/sysml/lops/AppendG.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
* TODO Additional compiler enhancements:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/AppendGAlignedSP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/AppendGAlignedSP.java b/src/main/java/org/apache/sysml/lops/AppendGAlignedSP.java
index b1b86c1..703a680 100644
--- a/src/main/java/org/apache/sysml/lops/AppendGAlignedSP.java
+++ b/src/main/java/org/apache/sysml/lops/AppendGAlignedSP.java
@@ -1,10 +1,10 @@
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class AppendGAlignedSP extends Lop
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/AppendM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/AppendM.java b/src/main/java/org/apache/sysml/lops/AppendM.java
index ab364ca..812e7d1 100644
--- a/src/main/java/org/apache/sysml/lops/AppendM.java
+++ b/src/main/java/org/apache/sysml/lops/AppendM.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
public class AppendM extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/AppendR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/AppendR.java b/src/main/java/org/apache/sysml/lops/AppendR.java
index 94b83bf..dd69607 100644
--- a/src/main/java/org/apache/sysml/lops/AppendR.java
+++ b/src/main/java/org/apache/sysml/lops/AppendR.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
public class AppendR extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Binary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Binary.java b/src/main/java/org/apache/sysml/lops/Binary.java
index 1ceabda..39104ef 100644
--- a/src/main/java/org/apache/sysml/lops/Binary.java
+++ b/src/main/java/org/apache/sysml/lops/Binary.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/BinaryM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/BinaryM.java b/src/main/java/org/apache/sysml/lops/BinaryM.java
index 5b03008..34ede22 100644
--- a/src/main/java/org/apache/sysml/lops/BinaryM.java
+++ b/src/main/java/org/apache/sysml/lops/BinaryM.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.Binary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.Binary.OperationTypes;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/BinaryScalar.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/BinaryScalar.java b/src/main/java/org/apache/sysml/lops/BinaryScalar.java
index 6fbe4a5..cdab353 100644
--- a/src/main/java/org/apache/sysml/lops/BinaryScalar.java
+++ b/src/main/java/org/apache/sysml/lops/BinaryScalar.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
* Lop to perform binary scalar operations. Both inputs must be scalars.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/BinaryUAggChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/BinaryUAggChain.java b/src/main/java/org/apache/sysml/lops/BinaryUAggChain.java
index d6b9171..92e4663 100644
--- a/src/main/java/org/apache/sysml/lops/BinaryUAggChain.java
+++ b/src/main/java/org/apache/sysml/lops/BinaryUAggChain.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
public class BinaryUAggChain extends Lop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CSVReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CSVReBlock.java b/src/main/java/org/apache/sysml/lops/CSVReBlock.java
index e856de6..b20c6b8 100644
--- a/src/main/java/org/apache/sysml/lops/CSVReBlock.java
+++ b/src/main/java/org/apache/sysml/lops/CSVReBlock.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CentralMoment.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CentralMoment.java b/src/main/java/org/apache/sysml/lops/CentralMoment.java
index b7ab685..ff9862c 100644
--- a/src/main/java/org/apache/sysml/lops/CentralMoment.java
+++ b/src/main/java/org/apache/sysml/lops/CentralMoment.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/Checkpoint.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Checkpoint.java b/src/main/java/org/apache/sysml/lops/Checkpoint.java
index e74699b..f9bf657 100644
--- a/src/main/java/org/apache/sysml/lops/Checkpoint.java
+++ b/src/main/java/org/apache/sysml/lops/Checkpoint.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import org.apache.spark.storage.StorageLevel;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CoVariance.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CoVariance.java b/src/main/java/org/apache/sysml/lops/CoVariance.java
index 2d1626e..62112af 100644
--- a/src/main/java/org/apache/sysml/lops/CoVariance.java
+++ b/src/main/java/org/apache/sysml/lops/CoVariance.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
* Lop to compute covariance between two 1D matrices
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CombineBinary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CombineBinary.java b/src/main/java/org/apache/sysml/lops/CombineBinary.java
index 81fafef..cedf0de 100644
--- a/src/main/java/org/apache/sysml/lops/CombineBinary.java
+++ b/src/main/java/org/apache/sysml/lops/CombineBinary.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashSet;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CombineTernary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CombineTernary.java b/src/main/java/org/apache/sysml/lops/CombineTernary.java
index 529d57b..0c53677 100644
--- a/src/main/java/org/apache/sysml/lops/CombineTernary.java
+++ b/src/main/java/org/apache/sysml/lops/CombineTernary.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashSet;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CombineUnary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CombineUnary.java b/src/main/java/org/apache/sysml/lops/CombineUnary.java
index da843e1..d5ac845 100644
--- a/src/main/java/org/apache/sysml/lops/CombineUnary.java
+++ b/src/main/java/org/apache/sysml/lops/CombineUnary.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
import java.util.HashSet;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CumulativeOffsetBinary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CumulativeOffsetBinary.java b/src/main/java/org/apache/sysml/lops/CumulativeOffsetBinary.java
index 290824a..3e530c5 100644
--- a/src/main/java/org/apache/sysml/lops/CumulativeOffsetBinary.java
+++ b/src/main/java/org/apache/sysml/lops/CumulativeOffsetBinary.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.Aggregate.OperationTypes;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/lops/CumulativePartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/CumulativePartialAggregate.java b/src/main/java/org/apache/sysml/lops/CumulativePartialAggregate.java
index 7590316..58c7060 100644
--- a/src/main/java/org/apache/sysml/lops/CumulativePartialAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/CumulativePartialAggregate.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.lops;
+package org.apache.sysml.lops;
-import com.ibm.bi.dml.lops.Aggregate.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
+import org.apache.sysml.lops.Aggregate.OperationTypes;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.*;
/**
[49/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/MLContext.java b/src/main/java/com/ibm/bi/dml/api/MLContext.java
deleted file mode 100644
index 39c6384..0000000
--- a/src/main/java/com/ibm/bi/dml/api/MLContext.java
+++ /dev/null
@@ -1,1396 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map.Entry;
-import java.util.Scanner;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.spark.SparkContext;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.rdd.RDD;
-
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.api.jmlc.JMLCUtils;
-import com.ibm.bi.dml.api.monitoring.SparkMonitoringUtil;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.hops.globalopt.GlobalOptimizerWrapper;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.rewrite.RewriteRemovePersistentReadWrite;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.IntIdentifier;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.StringIdentifier;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDProperties;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ConvertStringToLongTextPair;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockPairFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyTextInputFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.SparkListener;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.utils.Explain.ExplainCounts;
-
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.SQLContext;
-
-/**
- * MLContext is useful for passing RDDs as input/output to SystemML. This API avoids the need to read/write
- * from HDFS (which is another way to pass inputs to SystemML).
- * <p>
- * Typical usage for MLContext is as follows:
- * <pre><code>
- * scala> import com.ibm.bi.dml.api.MLContext
- * </code></pre>
- * <p>
- * Create input DataFrame from CSV file and potentially perform some feature transformation
- * <pre><code>
- * scala> val W = sqlContext.load("com.databricks.spark.csv", Map("path" -> "W.csv", "header" -> "false"))
- * scala> val H = sqlContext.load("com.databricks.spark.csv", Map("path" -> "H.csv", "header" -> "false"))
- * scala> val V = sqlContext.load("com.databricks.spark.csv", Map("path" -> "V.csv", "header" -> "false"))
- * </code></pre>
- * <p>
- * Create MLContext
- * <pre><code>
- * scala> val ml = new MLContext(sc)
- * </code></pre>
- * <p>
- * Register input and output DataFrame/RDD
- * Supported format:
- * <ol>
- * <li> DataFrame
- * <li> CSV/Text (as JavaRDD<String> or JavaPairRDD<LongWritable, Text>)
- * <li> Binary blocked RDD (JavaPairRDD<MatrixIndexes,MatrixBlock>))
- * </ol>
- * Also overloaded to support metadata information such as format, rlen, clen, ...
- * Please note the variable names given below in quotes correspond to the variables in DML script.
- * These variables need to have corresponding read/write associated in DML script.
- * Currently, only matrix variables are supported through registerInput/registerOutput interface.
- * To pass scalar variables, use named/positional arguments (described later) or wrap them into matrix variable.
- * <pre><code>
- * scala> ml.registerInput("V", V)
- * scala> ml.registerInput("W", W)
- * scala> ml.registerInput("H", H)
- * scala> ml.registerOutput("H")
- * scala> ml.registerOutput("W")
- * </code></pre>
- * <p>
- * Call script with default arguments:
- * <pre><code>
- * scala> val outputs = ml.execute("GNMF.dml")
- * </code></pre>
- * <p>
- * Also supported: calling script with positional arguments (args) and named arguments (nargs):
- * <pre><code>
- * scala> val args = Array("V.mtx", "W.mtx", "H.mtx", "2000", "1500", "50", "1", "WOut.mtx", "HOut.mtx")
- * scala> val nargs = Map("maxIter"->"1", "V" -> "")
- * scala> val outputs = ml.execute("GNMF.dml", args) # or ml.execute("GNMF_namedArgs.dml", nargs)
- * </code></pre>
- * <p>
- * To run the script again using different (or even same arguments), but using same registered input/outputs:
- * <pre><code>
- * scala> val new_outputs = ml.execute("GNMF.dml", new_args)
- * </code></pre>
- * <p>
- * However, to register new input/outputs, you need to first reset MLContext
- * <pre><code>
- * scala> ml.reset()
- * scala> ml.registerInput("V", newV)
- * </code></pre>
- * <p>
- * Experimental API:
- * To monitor performance (only supported for Spark 1.4.0 or higher),
- * <pre><code>
- * scala> val ml = new MLContext(sc, true)
- * </code></pre>
- * <p>
- * If monitoring performance is enabled,
- * <pre><code>
- * scala> print(ml.getMonitoringUtil().getExplainOutput())
- * scala> ml.getMonitoringUtil().getRuntimeInfoInHTML("runtime.html")
- * </code></pre>
- * <p>
- * Note: The execute(...) methods does not support parallel calls from same or different MLContext.
- * This is because current SystemML engine does not allow multiple invocation in same JVM.
- * So, if you plan to create a system which potentially creates multiple MLContext,
- * it is recommended to guard the execute(...) call using
- * <pre><code>
- * synchronized(MLContext.class) { ml.execute(...); }
- * </code></pre>
- */
-public class MLContext {
-
- // ----------------------------------------------------
- // TODO: To make MLContext multi-threaded, track getCurrentMLContext and also all singletons and
- // static variables in SystemML codebase.
- private static MLContext _activeMLContext = null;
-
- // Package protected so as to maintain a clean public API for MLContext.
- // Use MLContextProxy.getActiveMLContext() if necessary
- static MLContext getActiveMLContext() {
- return _activeMLContext;
- }
- // ----------------------------------------------------
-
- private SparkContext _sc = null; // Read while creating SystemML's spark context
- public SparkContext getSparkContext() {
- if(_sc == null) {
- throw new RuntimeException("No spark context set in MLContext");
- }
- return _sc;
- }
- private ArrayList<String> _inVarnames = null;
- private ArrayList<String> _outVarnames = null;
- private LocalVariableMap _variables = null; // temporary symbol table
- private Program _rtprog = null;
-
- // --------------------------------------------------
- // _monitorUtils is set only when MLContext(sc, true)
- private SparkMonitoringUtil _monitorUtils = null;
-
- /**
- * Experimental API. Not supported in Python MLContext API.
- * @return
- */
- public SparkMonitoringUtil getMonitoringUtil() {
- return _monitorUtils;
- }
- // --------------------------------------------------
-
- /**
- * Create an associated MLContext for given spark session.
- * @param sc
- * @throws DMLRuntimeException
- */
- public MLContext(SparkContext sc) throws DMLRuntimeException {
- initializeSpark(sc, false, false);
- }
-
- /**
- * Create an associated MLContext for given spark session.
- * @param sc
- * @throws DMLRuntimeException
- */
- public MLContext(JavaSparkContext sc) throws DMLRuntimeException {
- initializeSpark(sc.sc(), false, false);
- }
-
- // ====================================================================================
- // Register input APIs
- // 1. DataFrame
-
- /**
- * Register DataFrame as input. DataFrame is assumed to be in row format and each cell can be converted into double
- * through Double.parseDouble(cell.toString()). This is suitable for passing dense matrices. For sparse matrices,
- * consider passing through text format (using JavaRDD<String>, format="text")
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param df
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, DataFrame df) throws DMLRuntimeException {
- registerInput(varName, df, false);
- }
-
- /**
- * Register DataFrame as input.
- * Current version doesnot support containsID=true.
- * Note: for Spark 1.4.0 or higher, registerInput(varName, df.sort("ID").drop("ID"), true) = registerInput(varName, df, false)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param df
- * @param containsID false if the DataFrame has an column ID which denotes the row ID.
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, DataFrame df, boolean containsID) throws DMLRuntimeException {
- MatrixCharacteristics mcOut = new MatrixCharacteristics();
- JavaPairRDD<MatrixIndexes, MatrixBlock> rdd = RDDConverterUtilsExt.dataFrameToBinaryBlock(new JavaSparkContext(_sc), df, mcOut, containsID);
- registerInput(varName, rdd, mcOut);
- }
-
- /**
- * Experimental API. Not supported in Python MLContext API.
- * @param varName
- * @param df
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, MLMatrix df) throws DMLRuntimeException {
- registerInput(varName, MLMatrix.getRDDLazily(df), df.mc);
- }
-
- // ------------------------------------------------------------------------------------
- // 2. CSV/Text: Usually JavaRDD<String>, but also supports JavaPairRDD<LongWritable, Text>
- /**
- * Register CSV/Text as inputs: Method for supplying csv file format properties, but without dimensions or nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param hasHeader
- * @param delim
- * @param fill
- * @param missingValue
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaRDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue) throws DMLRuntimeException {
- registerInput(varName, rdd, format, hasHeader, delim, fill, missingValue, -1, -1, -1);
- }
-
- /**
- * Register CSV/Text as inputs: Method for supplying csv file format properties, but without dimensions or nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param hasHeader
- * @param delim
- * @param fill
- * @param missingValue
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, RDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, missingValue, -1, -1, -1);
- }
-
- /**
- * Register CSV/Text as inputs: Method for supplying csv file format properties along with dimensions or nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param hasHeader
- * @param delim
- * @param fill
- * @param missingValue
- * @param rlen
- * @param clen
- * @param nnz
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, RDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD(), format, hasHeader, delim, fill, missingValue, -1, -1, -1);
- }
-
- /**
- * Register CSV/Text as inputs: Method for supplying csv file format properties along with dimensions or nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param hasHeader
- * @param delim
- * @param fill
- * @param missingValue
- * @param rlen
- * @param clen
- * @param nnz
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaRDD<String> rdd, String format, boolean hasHeader,
- String delim, boolean fill, double missingValue, long rlen, long clen, long nnz) throws DMLRuntimeException {
- RDDProperties properties = new RDDProperties();
- properties.setHasHeader(hasHeader);
- properties.setFill(fill);
- properties.setDelim(delim);
- properties.setMissingValue(missingValue);
- registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, nnz, properties);
- }
-
- /**
- * Register CSV/Text as inputs: Convenience method without dimensions and nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, RDD<String> rdd, String format) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD().mapToPair(new ConvertStringToLongTextPair()), format, -1, -1, -1, null);
- }
-
- /**
- * Register CSV/Text as inputs: Convenience method without dimensions and nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaRDD<String> rdd, String format) throws DMLRuntimeException {
- registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, -1, -1, -1, null);
- }
-
- /**
- * Register CSV/Text as inputs: Convenience method with dimensions and but no nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param rlen
- * @param clen
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaRDD<String> rdd, String format, long rlen, long clen) throws DMLRuntimeException {
- registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, -1, null);
- }
-
- /**
- * Register CSV/Text as inputs: Convenience method with dimensions and but no nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param rlen
- * @param clen
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, RDD<String> rdd, String format, long rlen, long clen) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD().mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, -1, null);
- }
-
- /**
- * Register CSV/Text as inputs: with dimensions and nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param rlen
- * @param clen
- * @param nnz
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaRDD<String> rdd, String format, long rlen, long clen, long nnz) throws DMLRuntimeException {
- registerInput(varName, rdd.mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, nnz, null);
- }
-
- /**
- * Register CSV/Text as inputs: with dimensions and nnz. It uses default file properties (example: delim, fill, ..)
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param format
- * @param rlen
- * @param clen
- * @param nnz
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, RDD<String> rdd, String format, long rlen, long clen, long nnz) throws DMLRuntimeException {
- registerInput(varName, rdd.toJavaRDD().mapToPair(new ConvertStringToLongTextPair()), format, rlen, clen, nnz, null);
- }
-
- // All CSV related methods call this ... It provides access to dimensions, nnz, file properties.
- private void registerInput(String varName, JavaPairRDD<LongWritable, Text> textOrCsv_rdd, String format, long rlen, long clen, long nnz, RDDProperties properties) throws DMLRuntimeException {
- if(!(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)) {
- throw new DMLRuntimeException("The registerInput functionality only supported for spark runtime. Please use MLContext(sc) instead of default constructor.");
- }
-
- if(_variables == null)
- _variables = new LocalVariableMap();
- if(_inVarnames == null)
- _inVarnames = new ArrayList<String>();
-
- MatrixObject mo = null;
- if(format.compareTo("csv") == 0) {
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
- mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
- }
- else if(format.compareTo("text") == 0) {
- if(rlen == -1 || clen == -1) {
- throw new DMLRuntimeException("The metadata is required in registerInput for format:" + format);
- }
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
- mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
- }
- else if(format.compareTo("mm") == 0) {
- // TODO: Handle matrix market
- throw new DMLRuntimeException("Matrixmarket format is not yet implemented in registerInput: " + format);
- }
- else {
-
- throw new DMLRuntimeException("Incorrect format in registerInput: " + format);
- }
-
- JavaPairRDD<LongWritable, Text> rdd = textOrCsv_rdd.mapToPair(new CopyTextInputFunction());
- if(properties != null) {
- mo.setRddProperties(properties);
- }
- mo.setRDDHandle(new RDDObject(rdd, varName));
- _variables.put(varName, mo);
- _inVarnames.add(varName);
- checkIfRegisteringInputAllowed();
- }
-
- // ------------------------------------------------------------------------------------
-
- // 3. Binary blocked RDD: Support JavaPairRDD<MatrixIndexes,MatrixBlock>
-
- /**
- * Register binary blocked RDD with given dimensions, default block sizes and no nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param rlen
- * @param clen
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, long rlen, long clen) throws DMLRuntimeException {
- registerInput(varName, rdd, rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- }
-
- /**
- * Register binary blocked RDD with given dimensions, given block sizes and no nnz
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, long rlen, long clen, int brlen, int bclen) throws DMLRuntimeException {
- registerInput(varName, rdd, rlen, clen, brlen, bclen, -1);
- }
-
-
- /**
- * Register binary blocked RDD with given dimensions, given block sizes and given nnz (preferred).
- * <p>
- * Marks the variable in the DML script as input variable.
- * Note that this expects a "varName = read(...)" statement in the DML script which through non-MLContext invocation
- * would have been created by reading a HDFS file.
- * @param varName
- * @param rdd
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param nnz
- * @throws DMLRuntimeException
- */
- public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, long rlen, long clen, int brlen, int bclen, long nnz) throws DMLRuntimeException {
- if(rlen == -1 || clen == -1) {
- throw new DMLRuntimeException("The metadata is required in registerInput for binary format");
- }
-
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, brlen, bclen, nnz);
- registerInput(varName, rdd, mc);
- }
-
- // All binary blocked method call this.
- public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, MatrixCharacteristics mc) throws DMLRuntimeException {
- if(_variables == null)
- _variables = new LocalVariableMap();
- if(_inVarnames == null)
- _inVarnames = new ArrayList<String>();
- // Bug in Spark is messing up blocks and indexes due to too eager reuse of data structures
- JavaPairRDD<MatrixIndexes, MatrixBlock> copyRDD = rdd.mapToPair( new CopyBlockPairFunction() );
-
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "temp", new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
- mo.setRDDHandle(new RDDObject(copyRDD, varName));
- _variables.put(varName, mo);
- _inVarnames.add(varName);
- checkIfRegisteringInputAllowed();
- }
-
- // =============================================================================================
-
- /**
- * Marks the variable in the DML script as output variable.
- * Note that this expects a "write(varName, ...)" statement in the DML script which through non-MLContext invocation
- * would have written the matrix to HDFS.
- * @param varName
- * @throws DMLRuntimeException
- */
- public void registerOutput(String varName) throws DMLRuntimeException {
- if(!(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)) {
- throw new DMLRuntimeException("The registerOutput functionality only supported for spark runtime. Please use MLContext(sc) instead of default constructor.");
- }
- if(_outVarnames == null)
- _outVarnames = new ArrayList<String>();
- _outVarnames.add(varName);
- if(_variables == null)
- _variables = new LocalVariableMap();
- }
-
- // =============================================================================================
-
- /**
- * Execute DML script by passing named arguments using specified config file.
- * @param dmlScriptFilePath the dml script can be in local filesystem or in HDFS
- * @param namedArgs
- * @param parsePyDML
- * @param configFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, HashMap<String, String> namedArgs, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException {
- String [] args = new String[namedArgs.size()];
- int i = 0;
- for(Entry<String, String> entry : namedArgs.entrySet()) {
- if(entry.getValue().trim().compareTo("") == 0)
- args[i] = entry.getKey() + "=\"" + entry.getValue() + "\"";
- else
- args[i] = entry.getKey() + "=" + entry.getValue();
- i++;
- }
- return compileAndExecuteScript(dmlScriptFilePath, args, true, parsePyDML, configFilePath);
- }
-
- /**
- * Execute DML script by passing named arguments using specified config file.
- * @param dmlScriptFilePath the dml script can be in local filesystem or in HDFS
- * @param namedArgs
- * @param configFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, HashMap<String, String> namedArgs, String configFilePath) throws IOException, DMLException, ParseException {
- String [] args = new String[namedArgs.size()];
- int i = 0;
- for(Entry<String, String> entry : namedArgs.entrySet()) {
- if(entry.getValue().trim().compareTo("") == 0)
- args[i] = entry.getKey() + "=\"" + entry.getValue() + "\"";
- else
- args[i] = entry.getKey() + "=" + entry.getValue();
- i++;
- }
-
- return compileAndExecuteScript(dmlScriptFilePath, args, true, false, configFilePath);
- }
-
- /**
- * Execute DML script by passing named arguments with default configuration.
- * @param dmlScriptFilePath the dml script can be in local filesystem or in HDFS
- * @param namedArgs
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, HashMap<String, String> namedArgs) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, namedArgs, false, null);
- }
-
- /**
- * Execute DML script by passing named arguments.
- * @param dmlScriptFilePath
- * @param namedArgs
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, scala.collection.immutable.Map<String, String> namedArgs) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, new HashMap<String, String>(scala.collection.JavaConversions.mapAsJavaMap(namedArgs)));
- }
-
- /**
- * Experimental: Execute PyDML script by passing named arguments if parsePyDML=true.
- * @param dmlScriptFilePath
- * @param namedArgs
- * @param parsePyDML
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, HashMap<String, String> namedArgs, boolean parsePyDML) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, namedArgs, parsePyDML, null);
- }
-
- /**
- * Experimental: Execute PyDML script by passing named arguments if parsePyDML=true.
- * @param dmlScriptFilePath
- * @param namedArgs
- * @param parsePyDML
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, scala.collection.immutable.Map<String, String> namedArgs, boolean parsePyDML) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, new HashMap<String, String>(scala.collection.JavaConversions.mapAsJavaMap(namedArgs)), parsePyDML);
- }
-
- /**
- * Execute DML script by passing positional arguments using specified config file
- * @param dmlScriptFilePath
- * @param args
- * @param configFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, String [] args, String configFilePath) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, args, false, configFilePath);
- }
-
- /**
- * Execute DML script by passing positional arguments using specified config file
- * This method is implemented for compatibility with Python MLContext.
- * Java/Scala users should use 'MLOutput execute(String dmlScriptFilePath, String [] args, String configFilePath)' instead as
- * equivalent scala collections (Seq/ArrayBuffer) is not implemented.
- * @param dmlScriptFilePath
- * @param args
- * @param configFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, ArrayList<String> args, String configFilePath) throws IOException, DMLException, ParseException {
- String [] argsArr = new String[args.size()];
- argsArr = args.toArray(argsArr);
- return execute(dmlScriptFilePath, argsArr, false, configFilePath);
- }
-
- /**
- * Execute DML script by passing positional arguments using default configuration
- * @param dmlScriptFilePath
- * @param args
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, String [] args) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, args, false, null);
- }
-
- /**
- * Execute DML script by passing positional arguments using default configuration.
- * This method is implemented for compatibility with Python MLContext.
- * Java/Scala users should use 'MLOutput execute(String dmlScriptFilePath, String [] args)' instead as
- * equivalent scala collections (Seq/ArrayBuffer) is not implemented.
- * @param dmlScriptFilePath
- * @param args
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, ArrayList<String> args) throws IOException, DMLException, ParseException {
- String [] argsArr = new String[args.size()];
- argsArr = args.toArray(argsArr);
- return execute(dmlScriptFilePath, argsArr, false, null);
- }
-
- /**
- * Experimental: Execute DML script by passing positional arguments if parsePyDML=true, using default configuration.
- * This method is implemented for compatibility with Python MLContext.
- * Java/Scala users should use 'MLOutput execute(String dmlScriptFilePath, String [] args, boolean parsePyDML)' instead as
- * equivalent scala collections (Seq/ArrayBuffer) is not implemented.
- * @param dmlScriptFilePath
- * @param args
- * @param parsePyDML
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, ArrayList<String> args, boolean parsePyDML) throws IOException, DMLException, ParseException {
- String [] argsArr = new String[args.size()];
- argsArr = args.toArray(argsArr);
- return execute(dmlScriptFilePath, argsArr, parsePyDML, null);
- }
-
- /**
- * Experimental: Execute DML script by passing positional arguments if parsePyDML=true, using specified config file.
- * This method is implemented for compatibility with Python MLContext.
- * Java/Scala users should use 'MLOutput execute(String dmlScriptFilePath, String [] args, boolean parsePyDML, String configFilePath)' instead as
- * equivalent scala collections (Seq/ArrayBuffer) is not implemented.
- * @param dmlScriptFilePath
- * @param args
- * @param parsePyDML
- * @param configFilePath
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, ArrayList<String> args, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException {
- String [] argsArr = new String[args.size()];
- argsArr = args.toArray(argsArr);
- return execute(dmlScriptFilePath, argsArr, parsePyDML, configFilePath);
- }
-
- /**
- * Experimental: Execute DML script by passing positional arguments if parsePyDML=true, using specified config file.
- * @param dmlScriptFilePath
- * @param args
- * @param parsePyDML
- * @param configFilePath
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, String [] args, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException {
- return compileAndExecuteScript(dmlScriptFilePath, args, false, parsePyDML, configFilePath);
- }
-
- /**
- * Experimental: Execute DML script by passing positional arguments if parsePyDML=true, using default configuration.
- * @param dmlScriptFilePath
- * @param args
- * @param parsePyDML
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, String [] args, boolean parsePyDML) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, args, parsePyDML, null);
- }
-
- /**
- * Execute DML script without any arguments using specified config path
- * @param dmlScriptFilePath
- * @param configFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, String configFilePath) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, false, configFilePath);
- }
-
- /**
- * Execute DML script without any arguments using default configuration.
- * @param dmlScriptFilePath
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, false, null);
- }
-
- /**
- * Experimental: Execute DML script without any arguments if parsePyDML=true, using specified config path.
- * @param dmlScriptFilePath
- * @param parsePyDML
- * @param configFilePath
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException {
- return compileAndExecuteScript(dmlScriptFilePath, null, false, parsePyDML, configFilePath);
- }
-
- /**
- * Experimental: Execute DML script without any arguments if parsePyDML=true, using default configuration.
- * @param dmlScriptFilePath
- * @param parsePyDML
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput execute(String dmlScriptFilePath, boolean parsePyDML) throws IOException, DMLException, ParseException {
- return execute(dmlScriptFilePath, parsePyDML, null);
- }
-
- // -------------------------------- Utility methods begins ----------------------------------------------------------
-
-
- /**
- * Call this method if you want to clear any RDDs set via registerInput, registerOutput.
- * This is required if ml.execute(..) has been called earlier and you want to call a new DML script.
- * @throws DMLRuntimeException
- */
- public void reset()
- throws DMLRuntimeException
- {
- //cleanup variables from bufferpool, incl evicted files
- //(otherwise memory leak because bufferpool holds references)
- CacheableData.cleanupCacheDir();
-
- //clear mlcontext state
- _inVarnames = null;
- _outVarnames = null;
- _variables = null;
- }
-
- /**
- * Used internally
- * @param source
- * @param target
- * @throws LanguageException
- */
- void setAppropriateVarsForRead(Expression source, String target)
- throws LanguageException
- {
- boolean isTargetRegistered = isRegisteredAsInput(target);
- boolean isReadExpression = (source instanceof DataExpression && ((DataExpression) source).isRead());
- if(isTargetRegistered && isReadExpression) {
- // Do not check metadata file for registered reads
- ((DataExpression) source).setCheckMetadata(false);
-
- MatrixObject mo = null;
- try {
- mo = getMatrixObject(target);
- int blp = source.getBeginLine(); int bcp = source.getBeginColumn();
- int elp = source.getEndLine(); int ecp = source.getEndColumn();
- ((DataExpression) source).addVarParam(DataExpression.READROWPARAM, new IntIdentifier(mo.getNumRows(), source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.READCOLPARAM, new IntIdentifier(mo.getNumColumns(), source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.READNUMNONZEROPARAM, new IntIdentifier(mo.getNnz(), source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.DATATYPEPARAM, new StringIdentifier("matrix", source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier("double", source.getFilename(), blp, bcp, elp, ecp));
-
- if(mo.getMetaData() instanceof MatrixFormatMetaData) {
- MatrixFormatMetaData metaData = (MatrixFormatMetaData) mo.getMetaData();
- if(metaData.getOutputInfo() == OutputInfo.CSVOutputInfo) {
- ((DataExpression) source).addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV, source.getFilename(), blp, bcp, elp, ecp));
- }
- else if(metaData.getOutputInfo() == OutputInfo.TextCellOutputInfo) {
- ((DataExpression) source).addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_TEXT, source.getFilename(), blp, bcp, elp, ecp));
- }
- else if(metaData.getOutputInfo() == OutputInfo.BinaryBlockOutputInfo) {
- ((DataExpression) source).addVarParam(DataExpression.ROWBLOCKCOUNTPARAM, new IntIdentifier(mo.getNumRowsPerBlock(), source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.COLUMNBLOCKCOUNTPARAM, new IntIdentifier(mo.getNumColumnsPerBlock(), source.getFilename(), blp, bcp, elp, ecp));
- ((DataExpression) source).addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_BINARY, source.getFilename(), blp, bcp, elp, ecp));
- }
- else {
- throw new LanguageException("Unsupported format through MLContext");
- }
- }
- } catch (DMLRuntimeException e) {
- throw new LanguageException(e);
- }
- }
- }
-
- /**
- * Used internally
- * @param tmp
- * @return
- */
- ArrayList<Instruction> performCleanupAfterRecompilation(ArrayList<Instruction> tmp) {
- String [] outputs = null;
- if(_outVarnames != null) {
- outputs = _outVarnames.toArray(new String[0]);
- }
- else {
- outputs = new String[0];
- }
-
- // No need to clean up entire program as this method is only called for last level program block
-// JMLCUtils.cleanupRuntimeProgram(_rtprog, outputs);
-
- for( int i=0; i<tmp.size(); i++ )
- {
- Instruction linst = tmp.get(i);
- if( linst instanceof VariableCPInstruction && ((VariableCPInstruction)linst).isRemoveVariable() )
- {
- VariableCPInstruction varinst = (VariableCPInstruction) linst;
- for( String var : outputs )
- if( varinst.isRemoveVariable(var) )
- {
- tmp.remove(i);
- i--;
- break;
- }
- }
- }
-
- return tmp;
- }
-
- // -------------------------------- Utility methods ends ----------------------------------------------------------
-
-
- // -------------------------------- Experimental API begins ----------------------------------------------------------
- /**
- * Experimental api:
- * Setting monitorPerformance to true adds additional overhead of storing state. So, use it only if necessary.
- * @param sc
- * @param monitorPerformance
- * @throws DMLRuntimeException
- */
- public MLContext(SparkContext sc, boolean monitorPerformance) throws DMLRuntimeException {
- initializeSpark(sc, monitorPerformance, false);
- }
-
- /**
- * Experimental api:
- * Setting monitorPerformance to true adds additional overhead of storing state. So, use it only if necessary.
- * @param sc
- * @param monitorPerformance
- * @throws DMLRuntimeException
- */
- public MLContext(JavaSparkContext sc, boolean monitorPerformance) throws DMLRuntimeException {
- initializeSpark(sc.sc(), monitorPerformance, false);
- }
-
- /**
- * Experimental api:
- * Setting monitorPerformance to true adds additional overhead of storing state. So, use it only if necessary.
- * @param sc
- * @param monitorPerformance
- * @param setForcedSparkExecType
- * @throws DMLRuntimeException
- */
- public MLContext(SparkContext sc, boolean monitorPerformance, boolean setForcedSparkExecType) throws DMLRuntimeException {
- initializeSpark(sc, monitorPerformance, setForcedSparkExecType);
- }
-
- /**
- * Experimental api:
- * Setting monitorPerformance to true adds additional overhead of storing state. So, use it only if necessary.
- * @param sc
- * @param monitorPerformance
- * @param setForcedSparkExecType
- * @throws DMLRuntimeException
- */
- public MLContext(JavaSparkContext sc, boolean monitorPerformance, boolean setForcedSparkExecType) throws DMLRuntimeException {
- initializeSpark(sc.sc(), monitorPerformance, setForcedSparkExecType);
- }
-
- // -------------------------------- Experimental API ends ----------------------------------------------------------
-
- // -------------------------------- Private methods begins ----------------------------------------------------------
- private boolean isRegisteredAsInput(String varName) {
- if(_inVarnames != null) {
- for(String v : _inVarnames) {
- if(v.compareTo(varName) == 0) {
- return true;
- }
- }
- }
- return false;
- }
-
- private MatrixObject getMatrixObject(String varName) throws DMLRuntimeException {
- if(_variables != null) {
- Data mo = _variables.get(varName);
- if(mo instanceof MatrixObject) {
- return (MatrixObject) mo;
- }
- else {
- throw new DMLRuntimeException("ERROR: Incorrect type");
- }
- }
- throw new DMLRuntimeException("ERROR: getMatrixObject not set for variable:" + varName);
- }
-
-
- private int compareVersion(String versionStr1, String versionStr2) {
- Scanner s1 = null;
- Scanner s2 = null;
- try {
- s1 = new Scanner(versionStr1); s1.useDelimiter("\\.");
- s2 = new Scanner(versionStr2); s2.useDelimiter("\\.");
- while(s1.hasNextInt() && s2.hasNextInt()) {
- int version1 = s1.nextInt();
- int version2 = s2.nextInt();
- if(version1 < version2) {
- return -1;
- } else if(version1 > version2) {
- return 1;
- }
- }
-
- if(s1.hasNextInt()) return 1;
- }
- finally {
- if(s1 != null) s1.close();
- if(s2 != null) s2.close();
- }
-
- return 0;
- }
-
- private void initializeSpark(SparkContext sc, boolean monitorPerformance, boolean setForcedSparkExecType) throws DMLRuntimeException {
- MLContextProxy.setActive(true);
-
- this._sc = sc;
-
- if(compareVersion(sc.version(), "1.3.0") < 0 ) {
- throw new DMLRuntimeException("Expected spark version >= 1.3.0 for running SystemML");
- }
-
- if(setForcedSparkExecType)
- DMLScript.rtplatform = RUNTIME_PLATFORM.SPARK;
- else
- DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
-
- if(monitorPerformance) {
- initializeSparkListener(sc);
- }
- }
-
- private void initializeSparkListener(SparkContext sc) throws DMLRuntimeException {
- if(compareVersion(sc.version(), "1.4.0") < 0 ) {
- throw new DMLRuntimeException("Expected spark version >= 1.4.0 for monitoring MLContext performance");
- }
- SparkListener sparkListener = new SparkListener(sc);
- _monitorUtils = new SparkMonitoringUtil(sparkListener);
- sc.addSparkListener(sparkListener);
- }
-
- /**
- * Experimental API. Not supported in Python MLContext API.
- * @param dmlScript
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLOutput executeScript(String dmlScript) throws IOException, DMLException, ParseException {
- return compileAndExecuteScript(dmlScript, null, false, false, false, null);
- }
-
- public MLOutput executeScript(String dmlScript, String configFilePath) throws IOException, DMLException, ParseException {
- return compileAndExecuteScript(dmlScript, null, false, false, false, configFilePath);
- }
-
- private void checkIfRegisteringInputAllowed() throws DMLRuntimeException {
- if(!(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)) {
- throw new DMLRuntimeException("ERROR: registerInput is only allowed for spark execution mode");
- }
- }
-
- private MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isNamedArgument, boolean isPyDML, String configFilePath) throws IOException, DMLException, ParseException {
- return compileAndExecuteScript(dmlScriptFilePath, args, true, isNamedArgument, isPyDML, configFilePath);
- }
-
- /**
- * All the execute() methods call this, which after setting appropriate input/output variables
- * calls _compileAndExecuteScript
- * We have explicitly synchronized this function because MLContext/SystemML does not yet support multi-threading.
- * @param dmlScriptFilePath
- * @param args
- * @param isNamedArgument
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- private synchronized MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isFile, boolean isNamedArgument, boolean isPyDML, String configFilePath) throws IOException, DMLException, ParseException {
- try {
- if(getActiveMLContext() != null) {
- throw new DMLRuntimeException("SystemML (and hence by definition MLContext) doesnot support parallel execute() calls from same or different MLContexts. "
- + "As a temporary fix, please do explicit synchronization, i.e. synchronized(MLContext.class) { ml.execute(...) } ");
- }
- else {
- // Set active MLContext.
- _activeMLContext = this;
- }
-
-
- if(_monitorUtils != null) {
- _monitorUtils.resetMonitoringData();
- }
-
- if(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) {
-
- HashMap<String, JavaPairRDD<MatrixIndexes,MatrixBlock>> retVal = null;
-
- // Depending on whether registerInput/registerOutput was called initialize the variables
- String[] inputs = null; String[] outputs = null;
- if(_inVarnames != null) {
- inputs = _inVarnames.toArray(new String[0]);
- }
- else {
- inputs = new String[0];
- }
- if(_outVarnames != null) {
- outputs = _outVarnames.toArray(new String[0]);
- }
- else {
- outputs = new String[0];
- }
- HashMap<String, MatrixCharacteristics> outMetadata = new HashMap<String, MatrixCharacteristics>();
-
- HashMap<String, String> argVals = DMLScript.createArgumentsMap(isNamedArgument, args);
-
- // Run the DML script
- ExecutionContext ec = executeUsingSimplifiedCompilationChain(dmlScriptFilePath, isFile, argVals, isPyDML, inputs, outputs, _variables, configFilePath);
-
- // Now collect the output
- if(_outVarnames != null) {
- if(_variables == null) {
- throw new DMLRuntimeException("The symbol table returned after executing the script is empty");
- }
-
- for( String ovar : _outVarnames ) {
- if( _variables.keySet().contains(ovar) ) {
- if(retVal == null) {
- retVal = new HashMap<String, JavaPairRDD<MatrixIndexes,MatrixBlock>>();
- }
- retVal.put(ovar, ((SparkExecutionContext) ec).getBinaryBlockRDDHandleForVariable(ovar));
- outMetadata.put(ovar, ((SparkExecutionContext) ec).getMatrixCharacteristics(ovar)); // For converting output to dataframe
- }
- else {
- throw new DMLException("Error: The variable " + ovar + " is not available as output after the execution of the DMLScript.");
- }
- }
- }
-
- return new MLOutput(retVal, outMetadata);
- }
- else {
- throw new DMLRuntimeException("Unsupported runtime:" + DMLScript.rtplatform.name());
- }
-
- }
- finally {
- // Reset active MLContext.
- _activeMLContext = null;
- }
- }
-
-
- /**
- * This runs the DML script and returns the ExecutionContext for the caller to extract the output variables.
- * The caller (which is compileAndExecuteScript) is expected to set inputSymbolTable with appropriate matrix representation (RDD, MatrixObject).
- *
- * @param dmlScriptFilePath
- * @param args
- * @param isNamedArgument
- * @param parsePyDML
- * @param inputs
- * @param outputs
- * @param inputSymbolTable
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- private ExecutionContext executeUsingSimplifiedCompilationChain(String dmlScriptFilePath, boolean isFile, HashMap<String, String> argVals, boolean parsePyDML,
- String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath) throws IOException, DMLException, ParseException {
- DMLConfig config = null;
- if(configFilePath == null) {
- config = new DMLConfig();
- }
- else {
- config = new DMLConfig(configFilePath);
- }
-
- ConfigurationManager.setConfig(config);
-
- String dmlScriptStr = null;
- if(isFile)
- dmlScriptStr = DMLScript.readDMLScript("-f", dmlScriptFilePath);
- else
- dmlScriptStr = DMLScript.readDMLScript("-s", dmlScriptFilePath);
-
- if(_monitorUtils != null) {
- _monitorUtils.setDMLString(dmlScriptStr);
- }
-
- DataExpression.REJECT_READ_UNKNOWN_SIZE = false;
-
- //simplified compilation chain
- _rtprog = null;
-
- //parsing
- AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
- DMLProgram prog = parser.parse(dmlScriptFilePath, dmlScriptStr, argVals);
- if(prog == null) {
- throw new ParseException("Couldnot parse the file:" + dmlScriptFilePath);
- }
-
- //language validate
- DMLTranslator dmlt = new DMLTranslator(prog);
- dmlt.liveVariableAnalysis(prog);
- dmlt.validateParseTree(prog);
-
- //hop construct/rewrite
- dmlt.constructHops(prog);
- dmlt.rewriteHopsDAG(prog);
-
- Explain.explain(prog);
-
- //rewrite persistent reads/writes
- if(inputSymbolTable != null) {
- RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs);
- ProgramRewriter rewriter2 = new ProgramRewriter(rewrite);
- rewriter2.rewriteProgramHopDAGs(prog);
- }
-
- //lop construct and runtime prog generation
- dmlt.constructLops(prog);
- _rtprog = prog.getRuntimeProgram(config);
-
- //optional global data flow optimization
- if(OptimizerUtils.isOptLevel(OptimizationLevel.O4_GLOBAL_TIME_MEMORY) ) {
- _rtprog = GlobalOptimizerWrapper.optimizeProgram(prog, _rtprog);
- }
-
- // launch SystemML appmaster not required as it is already launched
-
- //count number compiled MR jobs / SP instructions
- ExplainCounts counts = Explain.countDistributedOperations(_rtprog);
- Statistics.resetNoOfCompiledJobs( counts.numJobs );
-
- // Initialize caching and scratch space
- DMLScript.initHadoopExecution(config);
-
- //final cleanup runtime prog
- JMLCUtils.cleanupRuntimeProgram(_rtprog, outputs);
-
- //create and populate execution context
- ExecutionContext ec = ExecutionContextFactory.createContext(_rtprog);
- if(inputSymbolTable != null) {
- ec.setVariables(inputSymbolTable);
- }
-
- // System.out.println(Explain.explain(_rtprog));
-
- //core execute runtime program
- _rtprog.execute( ec );
-
- if(_monitorUtils != null)
- _monitorUtils.setExplainOutput(Explain.explain(_rtprog));
-
- return ec;
- }
-
- // -------------------------------- Private methods ends ----------------------------------------------------------
-
- // TODO: Add additional create to provide sep, missing values, etc. for CSV
- /**
- * Experimental API: Might be discontinued in future release
- * @param sqlContext
- * @param filePath
- * @param format
- * @return
- * @throws IOException
- * @throws DMLException
- * @throws ParseException
- */
- public MLMatrix read(SQLContext sqlContext, String filePath, String format) throws IOException, DMLException, ParseException {
- this.reset();
- this.registerOutput("output");
- MLOutput out = this.executeScript("output = read(\"" + filePath + "\", format=\"" + format + "\"); " + MLMatrix.writeStmt);
- JavaPairRDD<MatrixIndexes, MatrixBlock> blocks = out.getBinaryBlockedRDD("output");
- MatrixCharacteristics mcOut = out.getMatrixCharacteristics("output");
- return MLMatrix.createMLMatrix(this, sqlContext, blocks, mcOut);
- }
-
-// // TODO: Test this in different scenarios: sparse/dense/mixed
-// /**
-// * Experimental unstable API: Might be discontinued in future release
-// * @param ml
-// * @param sqlContext
-// * @param mllibMatrix
-// * @return
-// * @throws DMLRuntimeException
-// */
-// public MLMatrix read(SQLContext sqlContext, BlockMatrix mllibMatrix) throws DMLRuntimeException {
-// long nnz = -1; // TODO: Find number of non-zeros from mllibMatrix ... This is important !!
-//
-// JavaPairRDD<Tuple2<Object, Object>, Matrix> mllibBlocks = JavaPairRDD.fromJavaRDD(mllibMatrix.blocks().toJavaRDD());
-// long rlen = mllibMatrix.numRows(); long clen = mllibMatrix.numCols();
-// int brlen = mllibMatrix.numRowBlocks();
-// int bclen = mllibMatrix.numColBlocks();
-// if(mllibMatrix.numRowBlocks() != DMLTranslator.DMLBlockSize && mllibMatrix.numColBlocks() != DMLTranslator.DMLBlockSize) {
-// System.err.println("WARNING: Since the block size of mllib matrix is not " + DMLTranslator.DMLBlockSize + ", it may cause "
-// + "reblocks");
-// }
-//
-// JavaPairRDD<MatrixIndexes, MatrixBlock> blocks = mllibBlocks
-// .mapToPair(new ConvertMLLibBlocksToBinaryBlocks(rlen, clen, brlen, bclen));
-//
-// MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, brlen, bclen, nnz);
-// return MLMatrix.createMLMatrix(this, sqlContext, blocks, mc);
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/MLContextProxy.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/MLContextProxy.java b/src/main/java/com/ibm/bi/dml/api/MLContextProxy.java
deleted file mode 100644
index ad32c5f..0000000
--- a/src/main/java/com/ibm/bi/dml/api/MLContextProxy.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.monitoring.Location;
-import com.ibm.bi.dml.parser.Expression;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-
-/**
- * The purpose of this proxy is to shield systemml internals from direct access to MLContext
- * which would try to load spark libraries and hence fail if these are not available. This
- * indirection is much more efficient than catching NoClassDefFoundErrors for every access
- * to MLContext (e.g., on each recompile).
- *
- */
-public class MLContextProxy
-{
-
- private static boolean _active = false;
-
- /**
- *
- * @param flag
- */
- public static void setActive(boolean flag) {
- _active = flag;
- }
-
- /**
- *
- * @return
- */
- public static boolean isActive() {
- return _active;
- }
-
- /**
- *
- * @param tmp
- */
- public static ArrayList<Instruction> performCleanupAfterRecompilation(ArrayList<Instruction> tmp)
- {
- if(MLContext.getActiveMLContext() != null) {
- return MLContext.getActiveMLContext().performCleanupAfterRecompilation(tmp);
- }
- return tmp;
- }
-
- /**
- *
- * @param source
- * @param targetname
- * @throws LanguageException
- */
- public static void setAppropriateVarsForRead(Expression source, String targetname)
- throws LanguageException
- {
- MLContext mlContext = MLContext.getActiveMLContext();
- if(mlContext != null) {
- mlContext.setAppropriateVarsForRead(source, targetname);
- }
- }
-
- public static MLContext getActiveMLContext() {
- return MLContext.getActiveMLContext();
- }
-
- public static void setInstructionForMonitoring(Instruction inst) {
- Location loc = inst.getLocation();
- MLContext mlContext = MLContext.getActiveMLContext();
- if(loc != null && mlContext != null && mlContext.getMonitoringUtil() != null) {
- mlContext.getMonitoringUtil().setInstructionLocation(loc, inst);
- }
- }
-
- public static void addRDDForInstructionForMonitoring(SPInstruction inst, Integer rddID) {
- MLContext mlContext = MLContext.getActiveMLContext();
- if(mlContext != null && mlContext.getMonitoringUtil() != null) {
- mlContext.getMonitoringUtil().addRDDForInstruction(inst, rddID);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/MLMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/MLMatrix.java b/src/main/java/com/ibm/bi/dml/api/MLMatrix.java
deleted file mode 100644
index 541f094..0000000
--- a/src/main/java/com/ibm/bi/dml/api/MLMatrix.java
+++ /dev/null
@@ -1,408 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package com.ibm.bi.dml.api;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.rdd.RDD;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.SQLContext.QueryExecution;
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
-import org.apache.spark.sql.types.StructType;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMIMBFromRow;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.GetMLBlock;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-
-/**
- * Experimental API: Might be discontinued in future release
- *
- * This class serves four purposes:
- * 1. It allows SystemML to fit nicely in MLPipeline by reducing number of reblocks.
- * 2. It allows users to easily read and write matrices without worrying
- * too much about format, metadata and type of underlying RDDs.
- * 3. It provides mechanism to convert to and from MLLib's BlockedMatrix format
- * 4. It provides off-the-shelf library for Distributed Blocked Matrix and reduces learning curve for using SystemML.
- * However, it is important to know that it is easy to abuse this off-the-shelf library and think it as replacement
- * to writing DML, which it is not. It does not provide any optimization between calls. A simple example
- * of the optimization that is conveniently skipped is: (t(m) %*% m)).
- * Also, note that this library is not thread-safe. The operator precedence is not exactly same as DML (as the precedence is
- * enforced by scala compiler), so please use appropriate brackets to enforce precedence.
-
- import com.ibm.bi.dml.api.{MLContext, MLMatrix}
- val ml = new MLContext(sc)
- val mat1 = ml.read(sqlContext, "V_small.csv", "csv")
- val mat2 = ml.read(sqlContext, "W_small.mtx", "binary")
- val result = mat1.transpose() %*% mat2
- result.write("Result_small.mtx", "text")
-
- */
-public class MLMatrix extends DataFrame {
- private static final long serialVersionUID = -7005940673916671165L;
- protected static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- protected MatrixCharacteristics mc = null;
- protected MLContext ml = null;
-
- protected MLMatrix(SQLContext sqlContext, LogicalPlan logicalPlan, MLContext ml) {
- super(sqlContext, logicalPlan);
- this.ml = ml;
- }
-
- protected MLMatrix(SQLContext sqlContext, QueryExecution queryExecution, MLContext ml) {
- super(sqlContext, queryExecution);
- this.ml = ml;
- }
-
- // Only used internally to set a new MLMatrix after one of matrix operations.
- // Not to be used externally.
- protected MLMatrix(DataFrame df, MatrixCharacteristics mc, MLContext ml) throws DMLRuntimeException {
- super(df.sqlContext(), df.logicalPlan());
- this.mc = mc;
- this.ml = ml;
- }
-
- static String writeStmt = "write(output, \"tmp\", format=\"binary\", rows_in_block=" + DMLTranslator.DMLBlockSize + ", cols_in_block=" + DMLTranslator.DMLBlockSize + ");";
-
- // ------------------------------------------------------------------------------------------------
-
-// /**
-// * Experimental unstable API: Converts our blocked matrix format to MLLib's format
-// * @return
-// */
-// public BlockMatrix toBlockedMatrix() {
-// JavaPairRDD<MatrixIndexes, MatrixBlock> blocks = getRDDLazily(this);
-// RDD<Tuple2<Tuple2<Object, Object>, Matrix>> mllibBlocks = blocks.mapToPair(new GetMLLibBlocks(mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock())).rdd();
-// return new BlockMatrix(mllibBlocks, mc.getRowsPerBlock(), mc.getColsPerBlock(), mc.getRows(), mc.getCols());
-// }
-
- // ------------------------------------------------------------------------------------------------
- static MLMatrix createMLMatrix(MLContext ml, SQLContext sqlContext, JavaPairRDD<MatrixIndexes, MatrixBlock> blocks, MatrixCharacteristics mc) throws DMLRuntimeException {
- RDD<Row> rows = blocks.map(new GetMLBlock()).rdd();
- StructType schema = MLBlock.getDefaultSchemaForBinaryBlock();
- return new MLMatrix(sqlContext.createDataFrame(rows.toJavaRDD(), schema), mc, ml);
- }
-
- /**
- * Convenient method to write a MLMatrix.
- */
- public void write(String filePath, String format) throws IOException, DMLException, ParseException {
- ml.reset();
- ml.registerInput("left", this);
- ml.executeScript("left = read(\"\"); output=left; write(output, \"" + filePath + "\", format=\"" + format + "\");");
- }
-
- private double getScalarBuiltinFunctionResult(String fn) throws IOException, DMLException, ParseException {
- if(fn.compareTo("nrow") == 0 || fn.compareTo("ncol") == 0) {
- ml.reset();
- ml.registerInput("left", getRDDLazily(this), mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), mc.getNonZeros());
- ml.registerOutput("output");
- String script = "left = read(\"\");"
- + "val = " + fn + "(left); "
- + "output = matrix(val, rows=1, cols=1); "
- + writeStmt;
- MLOutput out = ml.executeScript(script);
- List<Tuple2<MatrixIndexes, MatrixBlock>> result = out.getBinaryBlockedRDD("output").collect();
- if(result == null || result.size() != 1) {
- throw new DMLRuntimeException("Error while computing the function: " + fn);
- }
- return result.get(0)._2.getValue(0, 0);
- }
- else {
- throw new DMLRuntimeException("The function " + fn + " is not yet supported in MLMatrix");
- }
- }
-
- /**
- * Gets or computes the number of rows.
- * @return
- * @throws ParseException
- * @throws DMLException
- * @throws IOException
- */
- public long numRows() throws IOException, DMLException, ParseException {
- if(mc.rowsKnown()) {
- return mc.getRows();
- }
- else {
- return (long) getScalarBuiltinFunctionResult("nrow");
- }
- }
-
- /**
- * Gets or computes the number of columns.
- * @return
- * @throws ParseException
- * @throws DMLException
- * @throws IOException
- */
- public long numCols() throws IOException, DMLException, ParseException {
- if(mc.colsKnown()) {
- return mc.getCols();
- }
- else {
- return (long) getScalarBuiltinFunctionResult("ncol");
- }
- }
-
- public int rowsPerBlock() {
- return mc.getRowsPerBlock();
- }
-
- public int colsPerBlock() {
- return mc.getColsPerBlock();
- }
-
- private String getScript(String binaryOperator) {
- return "left = read(\"\");"
- + "right = read(\"\");"
- + "output = left " + binaryOperator + " right; "
- + writeStmt;
- }
-
- private String getScalarBinaryScript(String binaryOperator, double scalar, boolean isScalarLeft) {
- if(isScalarLeft) {
- return "left = read(\"\");"
- + "output = " + scalar + " " + binaryOperator + " left ;"
- + writeStmt;
- }
- else {
- return "left = read(\"\");"
- + "output = left " + binaryOperator + " " + scalar + ";"
- + writeStmt;
- }
- }
-
- static JavaPairRDD<MatrixIndexes, MatrixBlock> getRDDLazily(MLMatrix mat) {
- return mat.rdd().toJavaRDD().mapToPair(new GetMIMBFromRow());
- }
-
- private MLMatrix matrixBinaryOp(MLMatrix that, String op) throws IOException, DMLException, ParseException {
-
- if(mc.getRowsPerBlock() != that.mc.getRowsPerBlock() || mc.getColsPerBlock() != that.mc.getColsPerBlock()) {
- throw new DMLRuntimeException("Incompatible block sizes: brlen:" + mc.getRowsPerBlock() + "!=" + that.mc.getRowsPerBlock() + " || bclen:" + mc.getColsPerBlock() + "!=" + that.mc.getColsPerBlock());
- }
-
- if(op.compareTo("%*%") == 0) {
- if(mc.getCols() != that.mc.getRows()) {
- throw new DMLRuntimeException("Dimensions mismatch:" + mc.getCols() + "!=" + that.mc.getRows());
- }
- }
- else {
- if(mc.getRows() != that.mc.getRows() || mc.getCols() != that.mc.getCols()) {
- throw new DMLRuntimeException("Dimensions mismatch:" + mc.getRows() + "!=" + that.mc.getRows() + " || " + mc.getCols() + "!=" + that.mc.getCols());
- }
- }
-
- ml.reset();
- ml.registerInput("left", this);
- ml.registerInput("right", that);
- ml.registerOutput("output");
- MLOutput out = ml.executeScript(getScript(op));
- RDD<Row> rows = out.getBinaryBlockedRDD("output").map(new GetMLBlock()).rdd();
- StructType schema = MLBlock.getDefaultSchemaForBinaryBlock();
- MatrixCharacteristics mcOut = out.getMatrixCharacteristics("output");
- return new MLMatrix(this.sqlContext().createDataFrame(rows.toJavaRDD(), schema), mcOut, ml);
- }
-
- private MLMatrix scalarBinaryOp(Double scalar, String op, boolean isScalarLeft) throws IOException, DMLException, ParseException {
- ml.reset();
- ml.registerInput("left", this);
- ml.registerOutput("output");
- MLOutput out = ml.executeScript(getScalarBinaryScript(op, scalar, isScalarLeft));
- RDD<Row> rows = out.getBinaryBlockedRDD("output").map(new GetMLBlock()).rdd();
- StructType schema = MLBlock.getDefaultSchemaForBinaryBlock();
- MatrixCharacteristics mcOut = out.getMatrixCharacteristics("output");
- return new MLMatrix(this.sqlContext().createDataFrame(rows.toJavaRDD(), schema), mcOut, ml);
- }
-
- // ---------------------------------------------------
- // Simple operator loading but doesnot utilize the optimizer
-
- public MLMatrix $greater(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, ">");
- }
-
- public MLMatrix $less(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "<");
- }
-
- public MLMatrix $greater$eq(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, ">=");
- }
-
- public MLMatrix $less$eq(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "<=");
- }
-
- public MLMatrix $eq$eq(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "==");
- }
-
- public MLMatrix $bang$eq(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "!=");
- }
-
- public MLMatrix $up(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "^");
- }
-
- public MLMatrix exp(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "^");
- }
-
- public MLMatrix $plus(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "+");
- }
-
- public MLMatrix add(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "+");
- }
-
- public MLMatrix $minus(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "-");
- }
-
- public MLMatrix minus(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "-");
- }
-
- public MLMatrix $times(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "*");
- }
-
- public MLMatrix elementWiseMultiply(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "*");
- }
-
- public MLMatrix $div(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "/");
- }
-
- public MLMatrix divide(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "/");
- }
-
- public MLMatrix $percent$div$percent(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%/%");
- }
-
- public MLMatrix integerDivision(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%/%");
- }
-
- public MLMatrix $percent$percent(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%%");
- }
-
- public MLMatrix modulus(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%%");
- }
-
- public MLMatrix $percent$times$percent(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%*%");
- }
-
- public MLMatrix multiply(MLMatrix that) throws IOException, DMLException, ParseException {
- return matrixBinaryOp(that, "%*%");
- }
-
- public MLMatrix transpose() throws IOException, DMLException, ParseException {
- ml.reset();
- ml.registerInput("left", this);
- ml.registerOutput("output");
- String script = "left = read(\"\");"
- + "output = t(left); "
- + writeStmt;
- MLOutput out = ml.executeScript(script);
- RDD<Row> rows = out.getBinaryBlockedRDD("output").map(new GetMLBlock()).rdd();
- StructType schema = MLBlock.getDefaultSchemaForBinaryBlock();
- MatrixCharacteristics mcOut = out.getMatrixCharacteristics("output");
- return new MLMatrix(this.sqlContext().createDataFrame(rows.toJavaRDD(), schema), mcOut, ml);
- }
-
- // TODO: For 'scalar op matrix' operations: Do implicit conversions
- public MLMatrix $plus(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "+", false);
- }
-
- public MLMatrix add(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "+", false);
- }
-
- public MLMatrix $minus(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "-", false);
- }
-
- public MLMatrix minus(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "-", false);
- }
-
- public MLMatrix $times(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "*", false);
- }
-
- public MLMatrix elementWiseMultiply(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "*", false);
- }
-
- public MLMatrix $div(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "/", false);
- }
-
- public MLMatrix divide(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "/", false);
- }
-
- public MLMatrix $greater(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, ">", false);
- }
-
- public MLMatrix $less(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "<", false);
- }
-
- public MLMatrix $greater$eq(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, ">=", false);
- }
-
- public MLMatrix $less$eq(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "<=", false);
- }
-
- public MLMatrix $eq$eq(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "==", false);
- }
-
- public MLMatrix $bang$eq(Double scalar) throws IOException, DMLException, ParseException {
- return scalarBinaryOp(scalar, "!=", false);
- }
-
-}
[31/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
deleted file mode 100644
index 4a7b2d4..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ /dev/null
@@ -1,1544 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.OpOp1;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Rule: Algebraic Simplifications. Simplifies binary expressions
- * in terms of two major purposes: (1) rewrite binary operations
- * to unary operations when possible (in CP this reduces the memory
- * estimate, in MR this allows map-only operations and hence prevents
- * unnecessary shuffle and sort) and (2) remove binary operations that
- * are in itself are unnecessary (e.g., *1 and /1).
- *
- */
-public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
-{
-
- private static final Log LOG = LogFactory.getLog(RewriteAlgebraicSimplificationStatic.class.getName());
-
- private static OpOp2[] LOOKUP_VALID_DISTRIBUTIVE_BINARY = new OpOp2[]{OpOp2.PLUS, OpOp2.MINUS};
- private static OpOp2[] LOOKUP_VALID_ASSOCIATIVE_BINARY = new OpOp2[]{OpOp2.PLUS, OpOp2.MULT};
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return roots;
-
- //one pass rewrite-descend (rewrite created pattern)
- for( Hop h : roots )
- rule_AlgebraicSimplification( h, false );
-
- Hop.resetVisitStatus(roots);
-
- //one pass descend-rewrite (for rollup)
- for( Hop h : roots )
- rule_AlgebraicSimplification( h, true );
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return root;
-
- //one pass rewrite-descend (rewrite created pattern)
- rule_AlgebraicSimplification( root, false );
-
- root.resetVisitStatus();
-
- //one pass descend-rewrite (for rollup)
- rule_AlgebraicSimplification( root, true );
-
- return root;
- }
-
-
- /**
- * Note: X/y -> X * 1/y would be useful because * cheaper than / and sparsesafe; however,
- * (1) the results would be not exactly the same (2 rounds instead of 1) and (2) it should
- * come before constant folding while the other simplifications should come after constant
- * folding. Hence, not applied yet.
- *
- * @throws HopsException
- */
- private void rule_AlgebraicSimplification(Hop hop, boolean descendFirst)
- throws HopsException
- {
- if(hop.getVisited() == Hop.VisitStatus.DONE)
- return;
-
- //recursively process children
- for( int i=0; i<hop.getInput().size(); i++)
- {
- Hop hi = hop.getInput().get(i);
-
- //process childs recursively first (to allow roll-up)
- if( descendFirst )
- rule_AlgebraicSimplification(hi, descendFirst); //see below
-
- //apply actual simplification rewrites (of childs incl checks)
- hi = removeUnnecessaryVectorizeOperation(hi); //e.g., matrix(1,nrow(X),ncol(X))/X -> 1/X
- hi = removeUnnecessaryBinaryOperation(hop, hi, i); //e.g., X*1 -> X (dep: should come after rm unnecessary vectorize)
- hi = fuseDatagenAndBinaryOperation(hop, hi, i); //e.g., rand(min=-1,max=1)*7 -> rand(min=-7,max=7)
- hi = fuseDatagenAndMinusOperation(hop, hi, i); //e.g., -(rand(min=-2,max=1)) -> rand(min=-1,max=2)
- hi = simplifyBinaryToUnaryOperation(hi); //e.g., X*X -> X^2 (pow2)
- hi = simplifyMultiBinaryToBinaryOperation(hi); //e.g., 1-X*Y -> X 1-* Y
- hi = simplifyDistributiveBinaryOperation(hop, hi, i);//e.g., (X-Y*X) -> (1-Y)*X
- hi = simplifyBushyBinaryOperation(hop, hi, i); //e.g., (X*(Y*(Z%*%v))) -> (X*Y)*(Z%*%v)
- hi = simplifyUnaryAggReorgOperation(hop, hi, i); //e.g., sum(t(X)) -> sum(X)
- hi = simplifyTransposedAppend(hop, hi, i); //e.g., t(cbind(t(A),t(B))) -> rbind(A,B);
- hi = fuseBinarySubDAGToUnaryOperation(hop, hi, i); //e.g., X*(1-X)-> sprop(X) || 1/(1+exp(-X)) -> sigmoid(X) || X*(X>0) -> selp(X)
- hi = simplifyTraceMatrixMult(hop, hi, i); //e.g., trace(X%*%Y)->sum(X*t(Y));
- hi = simplifySlicedMatrixMult(hop, hi, i); //e.g., (X%*%Y)[1,1] -> X[1,] %*% Y[,1];
- hi = simplifyConstantSort(hop, hi, i); //e.g., order(matrix())->matrix/seq;
- hi = simplifyOrderedSort(hop, hi, i); //e.g., order(matrix())->seq;
- hi = removeUnnecessaryTranspose(hop, hi, i); //e.g., t(t(X))->X; potentially introduced by diag/trace_MM
- hi = removeUnnecessaryMinus(hop, hi, i); //e.g., -(-X)->X; potentially introduced by simplfiy binary or dyn rewrites
- hi = simplifyGroupedAggregate(hi); //e.g., aggregate(target=X,groups=y,fn="count") -> aggregate(target=y,groups=y,fn="count")
- hi = fuseMinusNzBinaryOperation(hop, hi, i); //e.g., X-mean*ppred(X,0,!=) -> X -nz mean
- hi = fuseLogNzBinaryOperation(hop, hi, i); //e.g., ppred(X,0,"!=")*log(X,0.5) -> log_nz(X,0.5)
- hi = simplifyOuterSeqExpand(hop, hi, i); //e.g., outer(v, seq(1,m), "==") -> rexpand(v, max=m, dir=row, ignore=true, cast=false)
- hi = simplifyTableSeqExpand(hop, hi, i); //e.g., table(seq(1,nrow(v)), v, nrow(v), m) -> rexpand(v, max=m, dir=row, ignore=false, cast=true)
- //hi = removeUnecessaryPPred(hop, hi, i); //e.g., ppred(X,X,"==")->matrix(1,rows=nrow(X),cols=ncol(X))
-
- //process childs recursively after rewrites (to investigate pattern newly created by rewrites)
- if( !descendFirst )
- rule_AlgebraicSimplification(hi, descendFirst);
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
-
- /**
- *
- * @param hi
- */
- private Hop removeUnnecessaryVectorizeOperation(Hop hi)
- {
- //applies to all binary matrix operations, if one input is unnecessarily vectorized
- if( hi instanceof BinaryOp && hi.getDataType()==DataType.MATRIX
- && ((BinaryOp)hi).supportsMatrixScalarOperations() )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- //NOTE: these rewrites of binary cell operations need to be aware that right is
- //potentially a vector but the result is of the size of left
- //TODO move to dynamic rewrites (since size dependent to account for mv binary cell and outer operations)
-
- if( !(left.getDim1()>1 && left.getDim2()==1 && right.getDim1()==1 && right.getDim2()>1) ) // no outer
- {
- //check and remove right vectorized scalar
- if( left.getDataType() == DataType.MATRIX && right instanceof DataGenOp )
- {
- DataGenOp dright = (DataGenOp) right;
- if( dright.getOp()==DataGenMethod.RAND && dright.hasConstantValue() )
- {
- Hop drightIn = dright.getInput().get(dright.getParamIndex(DataExpression.RAND_MIN));
- HopRewriteUtils.removeChildReference(bop, dright);
- HopRewriteUtils.addChildReference(bop, drightIn, 1);
- //cleanup if only consumer of intermediate
- if( dright.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( dright );
-
- LOG.debug("Applied removeUnnecessaryVectorizeOperation1");
- }
- }
- //check and remove left vectorized scalar
- else if( right.getDataType() == DataType.MATRIX && left instanceof DataGenOp )
- {
- DataGenOp dleft = (DataGenOp) left;
- if( dleft.getOp()==DataGenMethod.RAND && dleft.hasConstantValue()
- && (left.getDim2()==1 || right.getDim2()>1)
- && (left.getDim1()==1 || right.getDim1()>1))
- {
- Hop dleftIn = dleft.getInput().get(dleft.getParamIndex(DataExpression.RAND_MIN));
- HopRewriteUtils.removeChildReference(bop, dleft);
- HopRewriteUtils.addChildReference(bop, dleftIn, 0);
- //cleanup if only consumer of intermediate
- if( dleft.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( dleft );
-
- LOG.debug("Applied removeUnnecessaryVectorizeOperation2");
- }
- }
-
- //Note: we applied this rewrite to at most one side in order to keep the
- //output semantically equivalent. However, future extensions might consider
- //to remove vectors from both side, compute the binary op on scalars and
- //finally feed it into a datagenop of the original dimensions.
- }
- }
-
- return hi;
- }
-
-
- /**
- * handle removal of unnecessary binary operations
- *
- * X/1 or X*1 or 1*X or X-0 -> X
- * -1*X or X*-1-> -X
- *
- * @param parent
- * @param hi
- * @param pos
- * @throws HopsException
- */
- private Hop removeUnnecessaryBinaryOperation( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
- //X/1 or X*1 -> X
- if( left.getDataType()==DataType.MATRIX
- && right instanceof LiteralOp && ((LiteralOp)right).getDoubleValue()==1.0 )
- {
- if( bop.getOp()==OpOp2.DIV || bop.getOp()==OpOp2.MULT )
- {
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, left, pos);
- hi = left;
-
- LOG.debug("Applied removeUnnecessaryBinaryOperation1 (line "+bop.getBeginLine()+")");
- }
- }
- //X-0 -> X
- else if( left.getDataType()==DataType.MATRIX
- && right instanceof LiteralOp && ((LiteralOp)right).getDoubleValue()==0.0 )
- {
- if( bop.getOp()==OpOp2.MINUS )
- {
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, left, pos);
- hi = left;
-
- LOG.debug("Applied removeUnnecessaryBinaryOperation2 (line "+bop.getBeginLine()+")");
- }
- }
- //1*X -> X
- else if( right.getDataType()==DataType.MATRIX
- && left instanceof LiteralOp && ((LiteralOp)left).getDoubleValue()==1.0 )
- {
- if( bop.getOp()==OpOp2.MULT )
- {
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, right, pos);
- hi = right;
-
- LOG.debug("Applied removeUnnecessaryBinaryOperation3 (line "+bop.getBeginLine()+")");
- }
- }
- //-1*X -> -X
- //note: this rewrite is necessary since the new antlr parser always converts
- //-X to -1*X due to mechanical reasons
- else if( right.getDataType()==DataType.MATRIX
- && left instanceof LiteralOp && ((LiteralOp)left).getDoubleValue()==-1.0 )
- {
- if( bop.getOp()==OpOp2.MULT )
- {
- bop.setOp(OpOp2.MINUS);
- HopRewriteUtils.removeChildReferenceByPos(bop, left, 0);
- HopRewriteUtils.addChildReference(bop, new LiteralOp(0), 0);
- hi = bop;
-
- LOG.debug("Applied removeUnnecessaryBinaryOperation4 (line "+bop.getBeginLine()+")");
- }
- }
- //X*-1 -> -X (see comment above)
- else if( left.getDataType()==DataType.MATRIX
- && right instanceof LiteralOp && ((LiteralOp)right).getDoubleValue()==-1.0 )
- {
- if( bop.getOp()==OpOp2.MULT )
- {
- bop.setOp(OpOp2.MINUS);
- HopRewriteUtils.removeChildReferenceByPos(bop, right, 1);
- HopRewriteUtils.addChildReference(bop, new LiteralOp(0), 0);
- hi = bop;
-
- LOG.debug("Applied removeUnnecessaryBinaryOperation5 (line "+bop.getBeginLine()+")");
- }
- }
- }
-
- return hi;
- }
-
- /**
- * handle removal of unnecessary binary operations over rand data
- *
- * rand*7 -> rand(min*7,max*7); rand+7 -> rand(min+7,max+7);
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop fuseDatagenAndBinaryOperation( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- //left input rand and hence output matrix double, right scalar literal
- if( left instanceof DataGenOp && ((DataGenOp)left).getOp()==DataGenMethod.RAND &&
- right instanceof LiteralOp )
- {
- DataGenOp inputGen = (DataGenOp)left;
- HashMap<String,Integer> params = inputGen.getParamIndexMap();
- Hop min = left.getInput().get(params.get(DataExpression.RAND_MIN));
- Hop max = left.getInput().get(params.get(DataExpression.RAND_MAX));
- double sval = ((LiteralOp)right).getDoubleValue();
-
- if( (bop.getOp()==OpOp2.MULT || bop.getOp()==OpOp2.PLUS)
- && min instanceof LiteralOp && max instanceof LiteralOp )
- {
- //create fused data gen operator
- DataGenOp gen = null;
- if( bop.getOp()==OpOp2.MULT )
- gen = HopRewriteUtils.copyDataGenOp(inputGen, sval, 0);
- else //if( bop.getOp()==OpOp2.PLUS )
- gen = HopRewriteUtils.copyDataGenOp(inputGen, 1, sval);
-
- //rewire parents
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, gen, pos);
-
- //propagate potentially updated nnz=0
- parent.refreshSizeInformation();
-
- hi = gen;
-
- LOG.debug("Applied fuseDatagenAndBinaryOperation1");
- }
- }
- //right input rand and hence output matrix double, left scalar literal
- else if( right instanceof DataGenOp && ((DataGenOp)right).getOp()==DataGenMethod.RAND &&
- left instanceof LiteralOp )
- {
- DataGenOp inputGen = (DataGenOp)right;
- HashMap<String,Integer> params = inputGen.getParamIndexMap();
- Hop min = right.getInput().get(params.get(DataExpression.RAND_MIN));
- Hop max = right.getInput().get(params.get(DataExpression.RAND_MAX));
- double sval = ((LiteralOp)left).getDoubleValue();
-
- if( (bop.getOp()==OpOp2.MULT || bop.getOp()==OpOp2.PLUS)
- && min instanceof LiteralOp && max instanceof LiteralOp )
- {
- //create fused data gen operator
- DataGenOp gen = null;
- if( bop.getOp()==OpOp2.MULT )
- gen = HopRewriteUtils.copyDataGenOp(inputGen, sval, 0);
- else //if( bop.getOp()==OpOp2.PLUS )
- gen = HopRewriteUtils.copyDataGenOp(inputGen, 1, sval);
-
- //rewire parents
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, gen, pos);
-
- //propagate potentially updated nnz=0
- parent.refreshSizeInformation();
-
- hi = gen;
-
- LOG.debug("Applied fuseDatagenAndBinaryOperation2");
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop fuseDatagenAndMinusOperation( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- if( right instanceof DataGenOp && ((DataGenOp)right).getOp()==DataGenMethod.RAND &&
- left instanceof LiteralOp && ((LiteralOp)left).getDoubleValue()==0.0 )
- {
- DataGenOp inputGen = (DataGenOp)right;
- HashMap<String,Integer> params = inputGen.getParamIndexMap();
- int ixMin = params.get(DataExpression.RAND_MIN);
- int ixMax = params.get(DataExpression.RAND_MAX);
- Hop min = right.getInput().get(ixMin);
- Hop max = right.getInput().get(ixMax);
-
- //apply rewrite under additional conditions (for simplicity)
- if( inputGen.getParent().size()==1
- && min instanceof LiteralOp && max instanceof LiteralOp )
- {
- //exchange and *-1 (special case 0 stays 0 instead of -0 for consistency)
- double newMinVal = (((LiteralOp)max).getDoubleValue()==0)?0:(-1 * ((LiteralOp)max).getDoubleValue());
- double newMaxVal = (((LiteralOp)min).getDoubleValue()==0)?0:(-1 * ((LiteralOp)min).getDoubleValue());
- Hop newMin = new LiteralOp(newMinVal);
- Hop newMax = new LiteralOp(newMaxVal);
-
- HopRewriteUtils.removeChildReferenceByPos(inputGen, min, ixMin);
- HopRewriteUtils.addChildReference(inputGen, newMin, ixMin);
- HopRewriteUtils.removeChildReferenceByPos(inputGen, max, ixMax);
- HopRewriteUtils.addChildReference(inputGen, newMax, ixMax);
-
- HopRewriteUtils.removeChildReference(parent, bop);
- HopRewriteUtils.addChildReference(parent, inputGen, pos);
- hi = inputGen;
-
- LOG.debug("Applied fuseDatagenAndMinusOperation");
- }
- }
- }
-
- return hi;
- }
-
- /**
- * handle simplification of binary operations
- * (relies on previous common subexpression elimination)
- *
- * X+X -> X*2 or X*X -> X^2
- */
- private Hop simplifyBinaryToUnaryOperation( Hop hi )
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
- if( left == right && left.getDataType()==DataType.MATRIX )
- {
- //note: we simplify this to unary operations first (less mem and better MR plan),
- //however, we later compile specific LOPS for X*2 and X^2
- if( bop.getOp()==OpOp2.PLUS ) //X+X -> X*2
- {
- bop.setOp(OpOp2.MULT);
- LiteralOp tmp = new LiteralOp(2);
- bop.getInput().remove(1);
- right.getParent().remove(bop);
- HopRewriteUtils.addChildReference(hi, tmp, 1);
-
- LOG.debug("Applied simplifyBinaryToUnaryOperation1");
- }
- else if ( bop.getOp()==OpOp2.MULT ) //X*X -> X^2
- {
- bop.setOp(OpOp2.POW);
- LiteralOp tmp = new LiteralOp(2);
- bop.getInput().remove(1);
- right.getParent().remove(bop);
- HopRewriteUtils.addChildReference(hi, tmp, 1);
-
- LOG.debug("Applied simplifyBinaryToUnaryOperation2");
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param hi
- * @return
- */
- private Hop simplifyMultiBinaryToBinaryOperation( Hop hi )
- {
- //pattern: 1-(X*Y) --> X 1-* Y (avoid intermediate)
- if( hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==OpOp2.MINUS
- && hi.getDataType() == DataType.MATRIX
- && hi.getInput().get(0) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)hi.getInput().get(0))==1
- && hi.getInput().get(1) instanceof BinaryOp
- && ((BinaryOp)hi.getInput().get(1)).getOp()==OpOp2.MULT
- && hi.getInput().get(1).getParent().size() == 1 ) //single consumer
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = hi.getInput().get(1).getInput().get(0);
- Hop right = hi.getInput().get(1).getInput().get(1);
-
- //set new binaryop type and rewire inputs
- bop.setOp(OpOp2.MINUS1_MULT);
- HopRewriteUtils.removeAllChildReferences(hi);
- HopRewriteUtils.addChildReference(bop, left);
- HopRewriteUtils.addChildReference(bop, right);
-
- LOG.debug("Applied simplifyMultiBinaryToBinaryOperation.");
- }
-
- return hi;
- }
-
- /**
- * (X-Y*X) -> (1-Y)*X, (Y*X-X) -> (Y-1)*X
- * (X+Y*X) -> (1+Y)*X, (Y*X+X) -> (Y+1)*X
- *
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyDistributiveBinaryOperation( Hop parent, Hop hi, int pos )
- {
-
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- //(X+Y*X) -> (1+Y)*X, (Y*X+X) -> (Y+1)*X
- //(X-Y*X) -> (1-Y)*X, (Y*X-X) -> (Y-1)*X
- boolean applied = false;
- if( left.getDataType()==DataType.MATRIX && right.getDataType()==DataType.MATRIX
- && HopRewriteUtils.isValidOp(bop.getOp(), LOOKUP_VALID_DISTRIBUTIVE_BINARY) )
- {
- Hop X = null; Hop Y = null;
- if( left instanceof BinaryOp && ((BinaryOp)left).getOp()==OpOp2.MULT ) //(Y*X-X) -> (Y-1)*X
- {
- Hop leftC1 = left.getInput().get(0);
- Hop leftC2 = left.getInput().get(1);
- //System.out.println("aOp2:"+((BinaryOp)left).getOp()+": "+leftC1.getName()+" "+leftC2.getName());
-
- if( leftC1.getDataType()==DataType.MATRIX && leftC2.getDataType()==DataType.MATRIX &&
- (right == leftC1 || right == leftC2) && leftC1 !=leftC2 ){ //any mult order
- X = right;
- Y = ( right == leftC1 ) ? leftC2 : leftC1;
- }
- if( X != null ){ //rewrite 'binary +/-'
- HopRewriteUtils.removeChildReference(parent, hi);
- LiteralOp literal = new LiteralOp(1);
- BinaryOp plus = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), bop.getOp(), Y, literal);
- HopRewriteUtils.refreshOutputParameters(plus, right);
- BinaryOp mult = new BinaryOp(left.getName(), left.getDataType(), left.getValueType(), OpOp2.MULT, plus, X);
- HopRewriteUtils.refreshOutputParameters(mult, left);
-
- HopRewriteUtils.addChildReference(parent, mult, pos);
- hi = mult;
- applied = true;
-
- LOG.debug("Applied simplifyDistributiveBinaryOperation1");
- }
- }
-
- if( !applied && right instanceof BinaryOp && ((BinaryOp)right).getOp()==OpOp2.MULT ) //(X-Y*X) -> (1-Y)*X
- {
- Hop rightC1 = right.getInput().get(0);
- Hop rightC2 = right.getInput().get(1);
- if( rightC1.getDataType()==DataType.MATRIX && rightC2.getDataType()==DataType.MATRIX &&
- (left == rightC1 || left == rightC2) && rightC1 !=rightC2 ){ //any mult order
- X = left;
- Y = ( left == rightC1 ) ? rightC2 : rightC1;
- }
- if( X != null ){ //rewrite '+/- binary'
- HopRewriteUtils.removeChildReference(parent, hi);
- LiteralOp literal = new LiteralOp(1);
- BinaryOp plus = new BinaryOp(left.getName(), left.getDataType(), left.getValueType(), bop.getOp(), literal, Y);
- HopRewriteUtils.refreshOutputParameters(plus, left);
- BinaryOp mult = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), OpOp2.MULT, plus, X);
- HopRewriteUtils.refreshOutputParameters(mult, right);
-
- HopRewriteUtils.addChildReference(parent, mult, pos);
- hi = mult;
-
- LOG.debug("Applied simplifyDistributiveBinaryOperation2");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- * (X*(Y*(Z%*%v))) -> (X*Y)*(Z%*%v)
- * (X+(Y+(Z%*%v))) -> (X+Y)+(Z%*%v)
- *
- * Note: Restriction ba() at leaf and root instead of data at leaf to not reorganize too
- * eagerly, which would loose additional rewrite potential. This rewrite has two goals
- * (1) enable XtwXv, and increase piggybacking potential by creating bushy trees.
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyBushyBinaryOperation( Hop parent, Hop hi, int pos )
- {
- if( hi instanceof BinaryOp && parent instanceof AggBinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
- OpOp2 op = bop.getOp();
-
- if( left.getDataType()==DataType.MATRIX && right.getDataType()==DataType.MATRIX &&
- HopRewriteUtils.isValidOp(op, LOOKUP_VALID_ASSOCIATIVE_BINARY) )
- {
- boolean applied = false;
-
- if( right instanceof BinaryOp )
- {
- BinaryOp bop2 = (BinaryOp)right;
- Hop left2 = bop2.getInput().get(0);
- Hop right2 = bop2.getInput().get(1);
- OpOp2 op2 = bop2.getOp();
-
- if( op==op2 && right2.getDataType()==DataType.MATRIX
- && (right2 instanceof AggBinaryOp) )
- {
- //(X*(Y*op()) -> (X*Y)*op()
- HopRewriteUtils.removeChildReference(parent, bop);
-
- BinaryOp bop3 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, op, left, left2);
- HopRewriteUtils.refreshOutputParameters(bop3, bop);
- BinaryOp bop4 = new BinaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, op, bop3, right2);
- HopRewriteUtils.refreshOutputParameters(bop4, bop2);
-
- HopRewriteUtils.addChildReference(parent, bop4, pos);
- hi = bop4;
-
- applied = true;
-
- LOG.debug("Applied simplifyBushyBinaryOperation1");
- }
- }
-
- if( !applied && left instanceof BinaryOp )
- {
- BinaryOp bop2 = (BinaryOp)left;
- Hop left2 = bop2.getInput().get(0);
- Hop right2 = bop2.getInput().get(1);
- OpOp2 op2 = bop2.getOp();
-
- if( op==op2 && left2.getDataType()==DataType.MATRIX
- && (left2 instanceof AggBinaryOp)
- && (right2.getDim2() > 1 || right.getDim2() == 1) //X not vector, or Y vector
- && (right2.getDim1() > 1 || right.getDim1() == 1) ) //X not vector, or Y vector
- {
- //((op()*X)*Y) -> op()*(X*Y)
- HopRewriteUtils.removeChildReference(parent, bop);
-
- BinaryOp bop3 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, op, right2, right);
- HopRewriteUtils.refreshOutputParameters(bop3, bop2);
- BinaryOp bop4 = new BinaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, op, left2, bop3);
- HopRewriteUtils.refreshOutputParameters(bop4, bop);
-
- HopRewriteUtils.addChildReference(parent, bop4, pos);
- hi = bop4;
-
- LOG.debug("Applied simplifyBushyBinaryOperation2");
- }
- }
- }
-
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyUnaryAggReorgOperation( Hop parent, Hop hi, int pos )
- {
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getDirection()==Direction.RowCol //full uagg
- && hi.getInput().get(0) instanceof ReorgOp ) //reorg operation
- {
- ReorgOp rop = (ReorgOp)hi.getInput().get(0);
- if( (rop.getOp()==ReOrgOp.TRANSPOSE || rop.getOp()==ReOrgOp.RESHAPE) //valid reorg
- && rop.getParent().size()==1 ) //uagg only reorg consumer
- {
- Hop input = rop.getInput().get(0);
- HopRewriteUtils.removeAllChildReferences(hi);
- HopRewriteUtils.removeAllChildReferences(rop);
- HopRewriteUtils.addChildReference(hi, input);
-
- LOG.debug("Applied simplifyUnaryAggReorgOperation");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyTransposedAppend( Hop parent, Hop hi, int pos )
- {
- //e.g., t(cbind(t(A),t(B))) --> rbind(A,B), t(rbind(t(A),t(B))) --> cbind(A,B)
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.TRANSPOSE //t() rooted
- && hi.getInput().get(0) instanceof BinaryOp
- && (((BinaryOp)hi.getInput().get(0)).getOp()==OpOp2.CBIND //append (cbind/rbind)
- || ((BinaryOp)hi.getInput().get(0)).getOp()==OpOp2.RBIND)
- && hi.getInput().get(0).getParent().size() == 1 ) //single consumers of append
- {
- BinaryOp bop = (BinaryOp)hi.getInput().get(0);
- if( bop.getInput().get(0) instanceof ReorgOp //both inputs transpose ops
- && ((ReorgOp)bop.getInput().get(0)).getOp()==ReOrgOp.TRANSPOSE
- && bop.getInput().get(1) instanceof ReorgOp
- && ((ReorgOp)bop.getInput().get(1)).getOp()==ReOrgOp.TRANSPOSE )
- {
- Hop left = bop.getInput().get(0).getInput().get(0);
- Hop right = bop.getInput().get(1).getInput().get(0);
-
- //rewire links from parent, transpose, and binary
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, bop, pos);
- HopRewriteUtils.removeAllChildReferences(hi);
- HopRewriteUtils.removeAllChildReferences(bop);
-
- //change append type (safe due to single parent check)
- if( bop.getOp()==OpOp2.CBIND )
- bop.setOp(OpOp2.RBIND);
- else
- bop.setOp(OpOp2.CBIND);
-
- //relink new childs to binary op
- HopRewriteUtils.addChildReference(bop, left, 0);
- HopRewriteUtils.addChildReference(bop, right, 1);
- bop.refreshSizeInformation();
-
- hi = bop;
-
- LOG.debug("Applied simplifyTransposedAppend (line "+hi.getBeginLine()+").");
- }
- }
-
- return hi;
- }
-
- /**
- * handle simplification of more complex sub DAG to unary operation.
- *
- * X*(1-X) -> sprop(X)
- * (1-X)*X -> sprop(X)
- * 1/(1+exp(-X)) -> sigmoid(X)
- *
- * @param hi
- * @throws HopsException
- */
- private Hop fuseBinarySubDAGToUnaryOperation( Hop parent, Hop hi, int pos )
- throws HopsException
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = hi.getInput().get(0);
- Hop right = hi.getInput().get(1);
-
- //sample proportion (sprop) operator
- if( bop.getOp() == OpOp2.MULT && left.getDataType()==DataType.MATRIX && right.getDataType()==DataType.MATRIX )
- {
- //by definition, either left or right or none applies.
- //note: if there are multiple consumers on the intermediate,
- //we follow the heuristic that redundant computation is more beneficial,
- //i.e., we still fuse but leave the intermediate for the other consumers
-
- if( left instanceof BinaryOp ) //(1-X)*X
- {
- BinaryOp bleft = (BinaryOp)left;
- Hop left1 = bleft.getInput().get(0);
- Hop left2 = bleft.getInput().get(1);
-
- if( left1 instanceof LiteralOp &&
- HopRewriteUtils.getDoubleValue((LiteralOp)left1)==1 &&
- left2 == right && bleft.getOp() == OpOp2.MINUS )
- {
- UnaryOp unary = HopRewriteUtils.createUnary(right, OpOp1.SPROP);
- HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
- HopRewriteUtils.addChildReference(parent, unary, pos);
-
- //cleanup if only consumer of intermediate
- if( bop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop);
- if( left.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(left);
-
- hi = unary;
-
- LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-sprop1");
- }
- }
- if( right instanceof BinaryOp ) //X*(1-X)
- {
- BinaryOp bright = (BinaryOp)right;
- Hop right1 = bright.getInput().get(0);
- Hop right2 = bright.getInput().get(1);
-
- if( right1 instanceof LiteralOp &&
- HopRewriteUtils.getDoubleValue((LiteralOp)right1)==1 &&
- right2 == left && bright.getOp() == OpOp2.MINUS )
- {
- UnaryOp unary = HopRewriteUtils.createUnary(left, OpOp1.SPROP);
- HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
- HopRewriteUtils.addChildReference(parent, unary, pos);
-
- //cleanup if only consumer of intermediate
- if( bop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop);
- if( left.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(right);
-
- hi = unary;
-
- LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-sprop2");
- }
- }
- }
- //sigmoid operator
- else if( bop.getOp() == OpOp2.DIV && left.getDataType()==DataType.SCALAR && right.getDataType()==DataType.MATRIX
- && left instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)left)==1 && right instanceof BinaryOp)
- {
- //note: if there are multiple consumers on the intermediate,
- //we follow the heuristic that redundant computation is more beneficial,
- //i.e., we still fuse but leave the intermediate for the other consumers
-
- BinaryOp bop2 = (BinaryOp)right;
- Hop left2 = bop2.getInput().get(0);
- Hop right2 = bop2.getInput().get(1);
-
- if( bop2.getOp() == OpOp2.PLUS && left2.getDataType()==DataType.SCALAR && right2.getDataType()==DataType.MATRIX
- && left2 instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)left2)==1 && right2 instanceof UnaryOp)
- {
- UnaryOp uop = (UnaryOp) right2;
- Hop uopin = uop.getInput().get(0);
-
- if( uop.getOp()==OpOp1.EXP )
- {
- UnaryOp unary = null;
-
- //Pattern 1: (1/(1 + exp(-X))
- if( uopin instanceof BinaryOp && ((BinaryOp)uopin).getOp()==OpOp2.MINUS )
- {
- BinaryOp bop3 = (BinaryOp) uopin;
- Hop left3 = bop3.getInput().get(0);
- Hop right3 = bop3.getInput().get(1);
-
- if( left3 instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)left3)==0 ) {
- unary = HopRewriteUtils.createUnary(right3, OpOp1.SIGMOID);
- }
- }
- //Pattern 2: (1/(1 + exp(X)), e.g., where -(-X) has been removed by
- //the 'remove unnecessary minus' rewrite --> reintroduce the minus
- else
- {
- BinaryOp minus = HopRewriteUtils.createMinus(uopin);
- unary = HopRewriteUtils.createUnary(minus, OpOp1.SIGMOID);
- }
-
- if( unary != null )
- {
- HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
- HopRewriteUtils.addChildReference(parent, unary, pos);
-
- //cleanup if only consumer of intermediate
- if( bop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop);
- if( bop2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop2);
- if( uop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(uop);
-
- hi = unary;
-
- LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-sigmoid1");
- }
- }
- }
- }
- //select positive (selp) operator
- if( bop.getOp() == OpOp2.MULT && left.getDataType()==DataType.MATRIX && right.getDataType()==DataType.MATRIX )
- {
- //by definition, either left or right or none applies.
- //note: if there are multiple consumers on the intermediate tmp=(X>0), it's still beneficial
- //to replace the X*tmp with selp(X) due to lower memory requirements and simply sparsity propagation
-
- if( left instanceof BinaryOp ) //(X>0)*X
- {
- BinaryOp bleft = (BinaryOp)left;
- Hop left1 = bleft.getInput().get(0);
- Hop left2 = bleft.getInput().get(1);
-
- if( left2 instanceof LiteralOp &&
- HopRewriteUtils.getDoubleValue((LiteralOp)left2)==0 &&
- left1 == right && bleft.getOp() == OpOp2.GREATER )
- {
- UnaryOp unary = HopRewriteUtils.createUnary(right, OpOp1.SELP);
- HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
- HopRewriteUtils.addChildReference(parent, unary, pos);
-
- //cleanup if only consumer of intermediate
- if( bop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop);
- if( left.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(left);
-
- hi = unary;
-
- LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-selp1");
- }
- }
- if( right instanceof BinaryOp ) //X*(X>0)
- {
- BinaryOp bright = (BinaryOp)right;
- Hop right1 = bright.getInput().get(0);
- Hop right2 = bright.getInput().get(1);
-
- if( right2 instanceof LiteralOp &&
- HopRewriteUtils.getDoubleValue((LiteralOp)right2)==0 &&
- right1 == left && bright.getOp() == OpOp2.GREATER )
- {
- UnaryOp unary = HopRewriteUtils.createUnary(left, OpOp1.SELP);
- HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
- HopRewriteUtils.addChildReference(parent, unary, pos);
-
- //cleanup if only consumer of intermediate
- if( bop.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(bop);
- if( left.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences(right);
-
- hi = unary;
-
- LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-selp2");
- }
- }
- }
- }
-
- return hi;
- }
-
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- */
- private Hop simplifyTraceMatrixMult(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getOp()==AggOp.TRACE ) //trace()
- {
- Hop hi2 = hi.getInput().get(0);
- if( hi2 instanceof AggBinaryOp && ((AggBinaryOp)hi2).isMatrixMultiply() ) //X%*%Y
- {
- Hop left = hi2.getInput().get(0);
- Hop right = hi2.getInput().get(1);
-
- //remove link from parent to diag
- HopRewriteUtils.removeChildReference(parent, hi);
-
- //remove links to inputs to matrix mult
- //removeChildReference(hi2, left);
- //removeChildReference(hi2, right);
-
- //create new operators (incl refresh size inside for transpose)
- ReorgOp trans = HopRewriteUtils.createTranspose(right);
- BinaryOp mult = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), OpOp2.MULT, left, trans);
- mult.setRowsInBlock(right.getRowsInBlock());
- mult.setColsInBlock(right.getColsInBlock());
- mult.refreshSizeInformation();
- AggUnaryOp sum = new AggUnaryOp(right.getName(), DataType.SCALAR, right.getValueType(), AggOp.SUM, Direction.RowCol, mult);
- sum.refreshSizeInformation();
-
- //rehang new subdag under parent node
- HopRewriteUtils.addChildReference(parent, sum, pos);
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- hi = sum;
-
- LOG.debug("Applied simplifyTraceMatrixMult");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifySlicedMatrixMult(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //e.g., (X%*%Y)[1,1] -> X[1,] %*% Y[,1]
- if( hi instanceof IndexingOp
- && ((IndexingOp)hi).getRowLowerEqualsUpper()
- && ((IndexingOp)hi).getColLowerEqualsUpper()
- && hi.getInput().get(0).getParent().size()==1 //rix is single mm consumer
- && hi.getInput().get(0) instanceof AggBinaryOp
- && ((AggBinaryOp)hi.getInput().get(0)).isMatrixMultiply() )
- {
- Hop mm = hi.getInput().get(0);
- Hop X = mm.getInput().get(0);
- Hop Y = mm.getInput().get(1);
- Hop rowExpr = hi.getInput().get(1); //rl==ru
- Hop colExpr = hi.getInput().get(3); //cl==cu
-
- HopRewriteUtils.removeAllChildReferences(mm);
-
- //create new indexing operations
- IndexingOp ix1 = new IndexingOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, X,
- rowExpr, rowExpr, new LiteralOp(1), HopRewriteUtils.createValueHop(X, false), true, false);
- HopRewriteUtils.setOutputBlocksizes(ix1, X.getRowsInBlock(), X.getColsInBlock());
- ix1.refreshSizeInformation();
- IndexingOp ix2 = new IndexingOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, Y,
- new LiteralOp(1), HopRewriteUtils.createValueHop(Y, true), colExpr, colExpr, false, true);
- HopRewriteUtils.setOutputBlocksizes(ix2, Y.getRowsInBlock(), Y.getColsInBlock());
- ix2.refreshSizeInformation();
-
- //rewire matrix mult over ix1 and ix2
- HopRewriteUtils.addChildReference(mm, ix1, 0);
- HopRewriteUtils.addChildReference(mm, ix2, 1);
- mm.refreshSizeInformation();
-
- hi = mm;
-
- LOG.debug("Applied simplifySlicedMatrixMult");
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyConstantSort(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //order(matrix(7), indexreturn=FALSE) -> matrix(7)
- //order(matrix(7), indexreturn=TRUE) -> seq(1,nrow(X),1)
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.SORT ) //order
- {
- Hop hi2 = hi.getInput().get(0);
-
- if( hi2 instanceof DataGenOp && ((DataGenOp)hi2).getOp()==DataGenMethod.RAND
- && ((DataGenOp)hi2).hasConstantValue()
- && hi.getInput().get(3) instanceof LiteralOp ) //known indexreturn
- {
- if( HopRewriteUtils.getBooleanValue((LiteralOp)hi.getInput().get(3)) )
- {
- //order(matrix(7), indexreturn=TRUE) -> seq(1,nrow(X),1)
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- Hop seq = HopRewriteUtils.createSeqDataGenOp(hi2);
- seq.refreshSizeInformation();
- HopRewriteUtils.addChildReference(parent, seq, pos);
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeChildReference(hi, hi2);
- hi = seq;
-
- LOG.debug("Applied simplifyConstantSort1.");
- }
- else
- {
- //order(matrix(7), indexreturn=FALSE) -> matrix(7)
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hi2, pos);
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeChildReference(hi, hi2);
- hi = hi2;
-
- LOG.debug("Applied simplifyConstantSort2.");
- }
- }
- }
-
- return hi;
- }
-
- private Hop simplifyOrderedSort(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //order(seq(2,N+1,1), indexreturn=FALSE) -> matrix(7)
- //order(seq(2,N+1,1), indexreturn=TRUE) -> seq(1,N,1)/seq(N,1,-1)
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.SORT ) //order
- {
- Hop hi2 = hi.getInput().get(0);
-
- if( hi2 instanceof DataGenOp && ((DataGenOp)hi2).getOp()==DataGenMethod.SEQ )
- {
- Hop incr = hi2.getInput().get(((DataGenOp)hi2).getParamIndex(Statement.SEQ_INCR));
- //check for known ascending ordering and known indexreturn
- if( incr instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)incr)==1
- && hi.getInput().get(2) instanceof LiteralOp //decreasing
- && hi.getInput().get(3) instanceof LiteralOp ) //indexreturn
- {
- if( HopRewriteUtils.getBooleanValue((LiteralOp)hi.getInput().get(3)) ) //IXRET, ASC/DESC
- {
- //order(seq(2,N+1,1), indexreturn=TRUE) -> seq(1,N,1)/seq(N,1,-1)
- boolean desc = HopRewriteUtils.getBooleanValue((LiteralOp)hi.getInput().get(2));
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- Hop seq = HopRewriteUtils.createSeqDataGenOp(hi2, !desc);
- seq.refreshSizeInformation();
- HopRewriteUtils.addChildReference(parent, seq, pos);
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeChildReference(hi, hi2);
- hi = seq;
-
- LOG.debug("Applied simplifyOrderedSort1.");
- }
- else if( !HopRewriteUtils.getBooleanValue((LiteralOp)hi.getInput().get(2)) ) //DATA, ASC
- {
- //order(seq(2,N+1,1), indexreturn=FALSE) -> seq(2,N+1,1)
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hi2, pos);
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeChildReference(hi, hi2);
- hi = hi2;
-
- LOG.debug("Applied simplifyOrderedSort2.");
- }
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- */
- private Hop removeUnnecessaryTranspose(Hop parent, Hop hi, int pos)
- {
- if( hi instanceof ReorgOp && ((ReorgOp)hi).getOp()==ReOrgOp.TRANSPOSE ) //first transpose
- {
- Hop hi2 = hi.getInput().get(0);
- if( hi2 instanceof ReorgOp && ((ReorgOp)hi2).getOp()==ReOrgOp.TRANSPOSE ) //second transpose
- {
- Hop hi3 = hi2.getInput().get(0);
- //remove unnecessary chain of t(t())
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, hi3, pos);
- hi = hi3;
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- LOG.debug("Applied removeUnecessaryTranspose");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop removeUnnecessaryMinus(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi.getDataType() == DataType.MATRIX && hi instanceof BinaryOp
- && ((BinaryOp)hi).getOp()==OpOp2.MINUS //first minus
- && hi.getInput().get(0) instanceof LiteralOp && ((LiteralOp)hi.getInput().get(0)).getDoubleValue()==0 )
- {
- Hop hi2 = hi.getInput().get(1);
- if( hi2.getDataType() == DataType.MATRIX && hi2 instanceof BinaryOp
- && ((BinaryOp)hi2).getOp()==OpOp2.MINUS //second minus
- && hi2.getInput().get(0) instanceof LiteralOp && ((LiteralOp)hi2.getInput().get(0)).getDoubleValue()==0 )
-
- {
- Hop hi3 = hi2.getInput().get(1);
- //remove unnecessary chain of -(-())
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, hi3, pos);
- hi = hi3;
-
- //cleanup if only consumer of intermediate
- if( hi.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi );
- if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- LOG.debug("Applied removeUnecessaryMinus");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param hi
- * @return
- */
- private Hop simplifyGroupedAggregate(Hop hi)
- {
- if( hi instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp)hi).getOp()==ParamBuiltinOp.GROUPEDAGG ) //aggregate
- {
- ParameterizedBuiltinOp phi = (ParameterizedBuiltinOp)hi;
-
- if( phi.isCountFunction() ) //aggregate(fn="count")
- {
- HashMap<String, Integer> params = phi.getParamIndexMap();
- int ix1 = params.get(Statement.GAGG_TARGET);
- int ix2 = params.get(Statement.GAGG_GROUPS);
-
- //check for unnecessary memory consumption for "count"
- if( ix1 != ix2 && phi.getInput().get(ix1)!=phi.getInput().get(ix2) )
- {
- Hop th = phi.getInput().get(ix1);
- Hop gh = phi.getInput().get(ix2);
-
- HopRewriteUtils.removeChildReference(hi, th);
- HopRewriteUtils.addChildReference(hi, gh, ix1);
-
- LOG.debug("Applied simplifyGroupedAggregateCount");
- }
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop fuseMinusNzBinaryOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //pattern X - (s * ppred(X,0,!=)) -> X -nz s
- //note: this is done as a hop rewrite in order to significantly reduce the
- //memory estimate for X - tmp if X is sparse
- if( hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==OpOp2.MINUS
- && hi.getInput().get(0).getDataType()==DataType.MATRIX
- && hi.getInput().get(1).getDataType()==DataType.MATRIX
- && hi.getInput().get(1) instanceof BinaryOp
- && ((BinaryOp)hi.getInput().get(1)).getOp()==OpOp2.MULT )
- {
- Hop X = hi.getInput().get(0);
- Hop s = hi.getInput().get(1).getInput().get(0);
- Hop pred = hi.getInput().get(1).getInput().get(1);
-
- if( s.getDataType()==DataType.SCALAR && pred.getDataType()==DataType.MATRIX
- && pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL
- && pred.getInput().get(0) == X //depend on common subexpression elimination
- && pred.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred.getInput().get(1))==0 )
- {
- Hop hnew = new BinaryOp("tmp", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MINUS_NZ, X, s);
- HopRewriteUtils.setOutputBlocksizes(hnew, hi.getRowsInBlock(), hi.getColsInBlock());
- hnew.refreshSizeInformation();
-
- //relink new hop into original position
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
-
- LOG.debug("Applied fuseMinusNzBinaryOperation (line "+hi.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop fuseLogNzBinaryOperation(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //pattern ppred(X,0,"!=")*log(X,0.5) -> log_nz(X,0.5)
- //note: this is done as a hop rewrite in order to significantly reduce the
- //memory estimate and to prevent dense intermediates if X is ultra sparse
- if( hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==OpOp2.MULT
- && hi.getInput().get(0).getDataType()==DataType.MATRIX
- && hi.getInput().get(1).getDataType()==DataType.MATRIX
- && hi.getInput().get(1) instanceof BinaryOp
- && ((BinaryOp)hi.getInput().get(1)).getOp()==OpOp2.LOG )
- {
- Hop pred = hi.getInput().get(0);
- Hop X = hi.getInput().get(1).getInput().get(0);
- Hop log = hi.getInput().get(1).getInput().get(1);
-
- if( pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL
- && pred.getInput().get(0) == X //depend on common subexpression elimination
- && pred.getInput().get(1) instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred.getInput().get(1))==0 )
- {
- Hop hnew = new BinaryOp("tmp", DataType.MATRIX, ValueType.DOUBLE, OpOp2.LOG_NZ, X, log);
- HopRewriteUtils.setOutputBlocksizes(hnew, hi.getRowsInBlock(), hi.getColsInBlock());
- hnew.refreshSizeInformation();
-
- //relink new hop into original position
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, hnew, pos);
- hi = hnew;
-
- LOG.debug("Applied fuseLogNzBinaryOperation (line "+hi.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyOuterSeqExpand(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //pattern: outer(v, t(seq(1,m)), "==") -> rexpand(v, max=m, dir=row, ignore=true, cast=false)
- //note: this rewrite supports both left/right sequence
-
- if( hi instanceof BinaryOp && ((BinaryOp)hi).isOuterVectorOperator()
- && ((BinaryOp)hi).getOp()==OpOp2.EQUAL )
- {
- if( ( hi.getInput().get(1) instanceof ReorgOp //pattern a: outer(v, t(seq(1,m)), "==")
- && ((ReorgOp) hi.getInput().get(1)).getOp()==ReOrgOp.TRANSPOSE
- && HopRewriteUtils.isBasic1NSequence(hi.getInput().get(1).getInput().get(0)))
- || HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0))) //pattern b: outer(seq(1,m), t(v) "==")
- {
- //determine variable parameters for pattern a/b
- boolean isPatternB = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0));
- boolean isTransposeRight = (hi.getInput().get(1) instanceof ReorgOp
- && ((ReorgOp) hi.getInput().get(1)).getOp()==ReOrgOp.TRANSPOSE);
- Hop trgt = isPatternB ? (isTransposeRight ?
- hi.getInput().get(1).getInput().get(0) : //get v from t(v)
- HopRewriteUtils.createTranspose(hi.getInput().get(1)) ) : //create v via t(v')
- hi.getInput().get(0); //get v directly
- Hop seq = isPatternB ?
- hi.getInput().get(0) : hi.getInput().get(1).getInput().get(0);
- String direction = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) ? "rows" : "cols";
-
- //setup input parameter hops
- HashMap<String,Hop> inputargs = new HashMap<String,Hop>();
- inputargs.put("target", trgt);
- inputargs.put("max", HopRewriteUtils.getBasic1NSequenceMaxLiteral(seq));
- inputargs.put("dir", new LiteralOp(direction));
- inputargs.put("ignore", new LiteralOp(true));
- inputargs.put("cast", new LiteralOp(false));
-
- //create new hop
- ParameterizedBuiltinOp pbop = new ParameterizedBuiltinOp("tmp", DataType.MATRIX, ValueType.DOUBLE,
- ParamBuiltinOp.REXPAND, inputargs);
- HopRewriteUtils.setOutputBlocksizes(pbop, hi.getRowsInBlock(), hi.getColsInBlock());
- pbop.refreshSizeInformation();
-
- //relink new hop into original position
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, pbop, pos);
- hi = pbop;
-
- LOG.debug("Applied simplifyOuterSeqExpand (line "+hi.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
- /**
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- private Hop simplifyTableSeqExpand(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- //pattern: table(seq(1,nrow(v)), v, nrow(v), m) -> rexpand(v, max=m, dir=row, ignore=false, cast=true)
- //note: this rewrite supports both left/right sequence
-
- if( hi instanceof TernaryOp && hi.getInput().size()==5
- && hi.getInput().get(3) instanceof LiteralOp && hi.getInput().get(4) instanceof LiteralOp )
- {
- if( (HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) &&
- hi.getInput().get(3) instanceof LiteralOp) //pattern a: table(seq(1,nrow(v)), v, nrow(v), m)
- ||(HopRewriteUtils.isBasic1NSequence(hi.getInput().get(1)) &&
- hi.getInput().get(2) instanceof LiteralOp) ) //pattern b: table(v, seq(1,nrow(v)), m, nrow(v))
- {
- //determine variable parameters for pattern a/b
- int ixTgt = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) ? 1 : 0;
- int ixMax = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) ? 4 : 3;
- String direction = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) ? "cols" : "rows";
-
- //setup input parameter hops
- HashMap<String,Hop> inputargs = new HashMap<String,Hop>();
- inputargs.put("target", hi.getInput().get(ixTgt));
- inputargs.put("max", hi.getInput().get(ixMax));
- inputargs.put("dir", new LiteralOp(direction));
- inputargs.put("ignore", new LiteralOp(false));
- inputargs.put("cast", new LiteralOp(true));
-
- //create new hop
- ParameterizedBuiltinOp pbop = new ParameterizedBuiltinOp("tmp", DataType.MATRIX, ValueType.DOUBLE,
- ParamBuiltinOp.REXPAND, inputargs);
- HopRewriteUtils.setOutputBlocksizes(pbop, hi.getRowsInBlock(), hi.getColsInBlock());
- pbop.refreshSizeInformation();
-
- //relink new hop into original position
- HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, pbop, pos);
- hi = pbop;
-
- LOG.debug("Applied simplifyTableSeqExpand (line "+hi.getBeginLine()+")");
- }
- }
-
- return hi;
- }
-
- /**
- * NOTE: currently disabled since this rewrite is INVALID in the
- * presence of NaNs (because (NaN!=NaN) is true).
- *
- * @param parent
- * @param hi
- * @param pos
- * @return
- * @throws HopsException
- */
- @SuppressWarnings("unused")
- private Hop removeUnecessaryPPred(Hop parent, Hop hi, int pos)
- throws HopsException
- {
- if( hi instanceof BinaryOp )
- {
- BinaryOp bop = (BinaryOp)hi;
- Hop left = bop.getInput().get(0);
- Hop right = bop.getInput().get(1);
-
- Hop datagen = null;
-
- //ppred(X,X,"==") -> matrix(1, rows=nrow(X),cols=nrow(Y))
- if( left==right && bop.getOp()==OpOp2.EQUAL || bop.getOp()==OpOp2.GREATEREQUAL || bop.getOp()==OpOp2.LESSEQUAL )
- datagen = HopRewriteUtils.createDataGenOp(left, 1);
-
- //ppred(X,X,"!=") -> matrix(0, rows=nrow(X),cols=nrow(Y))
- if( left==right && bop.getOp()==OpOp2.NOTEQUAL || bop.getOp()==OpOp2.GREATER || bop.getOp()==OpOp2.LESS )
- datagen = HopRewriteUtils.createDataGenOp(left, 0);
-
- if( datagen != null )
- {
- HopRewriteUtils.removeChildReference(parent, hi);
- HopRewriteUtils.addChildReference(parent, datagen, pos);
- hi = datagen;
- }
- }
-
- return hi;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteBlockSizeAndReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteBlockSizeAndReblock.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteBlockSizeAndReblock.java
deleted file mode 100644
index bb4260e..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteBlockSizeAndReblock.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.Expression.DataType;
-
-/**
- * Rule: BlockSizeAndReblock. For all statement blocks, determine
- * "optimal" block size, and place reblock Hops. For now, we just
- * use BlockSize 1K x 1K and do reblock after Persistent Reads and
- * before Persistent Writes.
- */
-public class RewriteBlockSizeAndReblock extends HopRewriteRule
-{
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- //maintain rewrite status
- if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
-
- //perform reblock and blocksize rewrite
- for( Hop h : roots )
- rule_BlockSizeAndReblock(h, DMLTranslator.DMLBlockSize);
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return null;
-
- //maintain rewrite status
- if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
-
- //perform reblock and blocksize rewrite
- rule_BlockSizeAndReblock(root, DMLTranslator.DMLBlockSize);
-
- return root;
- }
-
- private void rule_BlockSizeAndReblock(Hop hop, final int GLOBAL_BLOCKSIZE)
- throws HopsException
- {
- // Go to the source(s) of the DAG
- for (Hop hi : hop.getInput()) {
- if (hi.getVisited() != Hop.VisitStatus.DONE)
- rule_BlockSizeAndReblock(hi, GLOBAL_BLOCKSIZE);
- }
-
- boolean canReblock = isReblockValid();
-
- if (hop instanceof DataOp)
- {
- // if block size does not match
- if( canReblock && hop.getDataType() == DataType.MATRIX
- && (hop.getRowsInBlock() != GLOBAL_BLOCKSIZE || hop.getColsInBlock() != GLOBAL_BLOCKSIZE) )
- {
- if (((DataOp) hop).getDataOpType() == DataOp.DataOpTypes.PERSISTENTREAD)
- {
- // insert reblock after the hop
- hop.setRequiresReblock(true);
- hop.setOutputBlocksizes(GLOBAL_BLOCKSIZE, GLOBAL_BLOCKSIZE);
- }
- else if (((DataOp) hop).getDataOpType() == DataOp.DataOpTypes.PERSISTENTWRITE)
- {
- if (hop.getRowsInBlock() == -1 && hop.getColsInBlock() == -1)
- {
- // if this dataop is for cell output, then no reblock is needed
- // as (A) all jobtypes can produce block2cell and cell2cell and
- // (B) we don't generate an explicit instruction for it (the info
- // is conveyed through OutputInfo.
- }
- else if (hop.getInput().get(0).requiresReblock() && hop.getInput().get(0).getParent().size() == 1)
- {
- // if a reblock is feeding into this, then use it if this is
- // the only parent, otherwise new Reblock
- hop.getInput().get(0).setOutputBlocksizes(hop.getRowsInBlock(),hop.getColsInBlock());
- }
- else
- {
- // insert reblock after the hop
- hop.setRequiresReblock(true);
- hop.setOutputBlocksizes(GLOBAL_BLOCKSIZE, GLOBAL_BLOCKSIZE);
- }
- }
- else if (((DataOp) hop).getDataOpType() == DataOp.DataOpTypes.TRANSIENTWRITE
- || ((DataOp) hop).getDataOpType() == DataOp.DataOpTypes.TRANSIENTREAD) {
- if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE ) {
- // simply copy the values from its input
- hop.setRowsInBlock(hop.getInput().get(0).getRowsInBlock());
- hop.setColsInBlock(hop.getInput().get(0).getColsInBlock());
- }
- else {
- // by default, all transient reads and writes are in blocked format
- hop.setRowsInBlock(GLOBAL_BLOCKSIZE);
- hop.setColsInBlock(GLOBAL_BLOCKSIZE);
- }
-
- } else {
- throw new HopsException(hop.printErrorLocation() + "unexpected non-scalar Data HOP in reblock.\n");
- }
- }
- }
- else if ( (hop instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp)hop).getOp() == ParamBuiltinOp.TRANSFORM) ) {
-
- // check if there exists a non-csv-write output. If yes, add reblock
- boolean rblk = false;
- for(Hop out : hop.getParent())
- {
- if ( !(out instanceof DataOp
- && ((DataOp)out).getDataOpType() == DataOpTypes.PERSISTENTWRITE
- && ((DataOp)out).getInputFormatType() == FileFormatTypes.CSV) )
- {
- rblk = true;
- break;
- }
- }
- if ( rblk )
- {
- hop.setRequiresReblock(true);
- hop.setOutputBlocksizes(GLOBAL_BLOCKSIZE, GLOBAL_BLOCKSIZE);
- }
- }
- else //NO DATAOP
- {
- // TODO: following two lines are commented, and the subsequent hack is used instead!
- //set_rows_per_block(GLOBAL_BLOCKSIZE);
- //set_cols_per_block(GLOBAL_BLOCKSIZE);
-
- /*
- * Handle hops whose output dimensions are unknown!
- *
- * Constraint C1:
- * Currently, only ctable() and groupedAggregate() fall into this category.
- * The MR jobs for both these functions run in "cell" mode and hence make their
- * blocking dimensions to (-1,-1).
- *
- * Constraint C2:
- * Blocking dimensions are not applicable for hops that produce scalars.
- * CMCOV and GroupedAgg jobs always run in "cell" mode, and hence they
- * produce output in cell format.
- *
- * Constraint C3:
- * Remaining hops will get their blocking dimensions from their input hops.
- */
-
- if ( hop.requiresReblock() ) {
- hop.setRowsInBlock(GLOBAL_BLOCKSIZE);
- hop.setColsInBlock(GLOBAL_BLOCKSIZE);
- }
-
- // Constraint C1:
-
- // Constraint C2:
- else if ( hop.getDataType() == DataType.SCALAR ) {
- hop.setRowsInBlock(-1);
- hop.setColsInBlock(-1);
- }
-
- // Constraint C3:
- else {
- if ( !canReblock ) {
- hop.setRowsInBlock(-1);
- hop.setColsInBlock(-1);
- }
- else {
- hop.setRowsInBlock(GLOBAL_BLOCKSIZE);
- hop.setColsInBlock(GLOBAL_BLOCKSIZE);
-
- // Functions may return multiple outputs, as defined in array of outputs in FunctionOp.
- // Reblock properties need to be set for each output.
- if ( hop instanceof FunctionOp ) {
- FunctionOp fop = (FunctionOp) hop;
- if ( fop.getOutputs() != null) {
- for(Hop out : fop.getOutputs()) {
- out.setRowsInBlock(GLOBAL_BLOCKSIZE);
- out.setColsInBlock(GLOBAL_BLOCKSIZE);
- }
- }
- }
- }
-
- // if any input is not blocked then the output of current Hop should not be blocked
- for ( Hop h : hop.getInput() ) {
- if ( h.getDataType() == DataType.MATRIX && h.getRowsInBlock() == -1 && h.getColsInBlock() == -1 ) {
- hop.setRowsInBlock(-1);
- hop.setColsInBlock(-1);
- break;
- }
- }
- }
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- }
-
- /**
- *
- * @return
- */
- private static boolean isReblockValid() {
- return ( DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteCommonSubexpressionElimination.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteCommonSubexpressionElimination.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteCommonSubexpressionElimination.java
deleted file mode 100644
index 9bb12f2..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteCommonSubexpressionElimination.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-
-/**
- * Rule: CommonSubexpressionElimination. For all statement blocks,
- * eliminate common subexpressions within dags by merging equivalent
- * operators (same input, equal parameters) bottom-up. For the moment,
- * this only applies within a dag, later this should be extended across
- * statements block (global, inter-procedure).
- */
-public class RewriteCommonSubexpressionElimination extends HopRewriteRule
-{
-
- private boolean _mergeLeafs = true;
-
- public RewriteCommonSubexpressionElimination()
- {
- this( true ); //default full CSE
- }
-
- public RewriteCommonSubexpressionElimination( boolean mergeLeafs )
- {
- _mergeLeafs = mergeLeafs;
- }
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- HashMap<String, Hop> dataops = new HashMap<String, Hop>();
- HashMap<String, Hop> literalops = new HashMap<String, Hop>(); //key: <VALUETYPE>_<LITERAL>
- for (Hop h : roots)
- {
- int cseMerged = 0;
- if( _mergeLeafs ) {
- cseMerged += rule_CommonSubexpressionElimination_MergeLeafs(h, dataops, literalops);
- h.resetVisitStatus();
- }
- cseMerged += rule_CommonSubexpressionElimination(h);
-
- if( cseMerged > 0 )
- LOG.debug("Common Subexpression Elimination - removed "+cseMerged+" operators.");
- }
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return null;
-
- HashMap<String, Hop> dataops = new HashMap<String, Hop>();
- HashMap<String, Hop> literalops = new HashMap<String, Hop>(); //key: <VALUETYPE>_<LITERAL>
- int cseMerged = 0;
- if( _mergeLeafs ) {
- cseMerged += rule_CommonSubexpressionElimination_MergeLeafs(root, dataops, literalops);
- root.resetVisitStatus();
- }
- cseMerged += rule_CommonSubexpressionElimination(root);
-
- if( cseMerged > 0 )
- LOG.debug("Common Subexpression Elimination - removed "+cseMerged+" operators.");
-
- return root;
- }
-
- /**
- *
- * @param dataops
- * @param literalops
- * @return
- * @throws HopsException
- */
- private int rule_CommonSubexpressionElimination_MergeLeafs( Hop hop, HashMap<String, Hop> dataops, HashMap<String, Hop> literalops )
- throws HopsException
- {
- int ret = 0;
- if( hop.getVisited() == Hop.VisitStatus.DONE )
- return ret;
-
- if( hop.getInput().isEmpty() ) //LEAF NODE
- {
- if( hop instanceof LiteralOp )
- {
- String key = hop.getValueType()+"_"+hop.getName();
- if( !literalops.containsKey(key) )
- literalops.put(key, hop);
- }
- else if( hop instanceof DataOp && ((DataOp)hop).isRead())
- {
- if(!dataops.containsKey(hop.getName()) )
- dataops.put(hop.getName(), hop);
- }
- }
- else //INNER NODE
- {
- //merge leaf nodes (data, literal)
- for( int i=0; i<hop.getInput().size(); i++ )
- {
- Hop hi = hop.getInput().get(i);
- String litKey = hi.getValueType()+"_"+hi.getName();
- if( hi instanceof DataOp && ((DataOp)hi).isRead() && dataops.containsKey(hi.getName()) )
- {
-
- //replace child node ref
- Hop tmp = dataops.get(hi.getName());
- if( tmp != hi ) { //if required
- tmp.getParent().add(hop);
- hop.getInput().set(i, tmp);
- ret++;
- }
- }
- else if( hi instanceof LiteralOp && literalops.containsKey(litKey) )
- {
- Hop tmp = literalops.get(litKey);
-
- //replace child node ref
- if( tmp != hi ){ //if required
- tmp.getParent().add(hop);
- hop.getInput().set(i, tmp);
- ret++;
- }
- }
-
- //recursive invocation (direct return on merged nodes)
- ret += rule_CommonSubexpressionElimination_MergeLeafs(hi, dataops, literalops);
- }
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
- return ret;
- }
-
- /**
- *
- * @param dataops
- * @param literalops
- * @return
- * @throws HopsException
- */
- private int rule_CommonSubexpressionElimination( Hop hop )
- throws HopsException
- {
- int ret = 0;
- if( hop.getVisited() == Hop.VisitStatus.DONE )
- return ret;
-
- //step 1: merge childs recursively first
- for(Hop hi : hop.getInput())
- ret += rule_CommonSubexpressionElimination(hi);
-
-
- //step 2: merge parent nodes
- if( hop.getParent().size()>1 ) //multiple consumers
- {
- //for all pairs
- for( int i=0; i<hop.getParent().size()-1; i++ )
- for( int j=i+1; j<hop.getParent().size(); j++ )
- {
- Hop h1 = hop.getParent().get(i);
- Hop h2 = hop.getParent().get(j);
-
- if( h1==h2 )
- {
- //do nothing, note: we should not remove redundant parent links
- //(otherwise rewrites would need to take this property into account)
-
- //remove redundant h2 from parent list
- //hop.getParent().remove(j);
- //j--;
- }
- else if( h1.compare(h2) ) //merge h2 into h1
- {
- //remove h2 from parent list
- hop.getParent().remove(j);
-
- //replace h2 w/ h1 in h2-parent inputs
- ArrayList<Hop> parent = h2.getParent();
- for( Hop p : parent )
- for( int k=0; k<p.getInput().size(); k++ )
- if( p.getInput().get(k)==h2 )
- {
- p.getInput().set(k, h1);
- h1.getParent().add(p);
- }
-
- //replace h2 w/ h1 in h2-input parents
- for( Hop in : h2.getInput() )
- in.getParent().remove(h2);
-
- ret++;
- j--;
- }
- }
- }
-
- hop.setVisited(Hop.VisitStatus.DONE);
-
- return ret;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteConstantFolding.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteConstantFolding.java
deleted file mode 100644
index 8b1bf5b..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteConstantFolding.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-
-/**
- * Rule: Constant Folding. For all statement blocks,
- * eliminate simple binary expressions of literals within dags by
- * computing them and replacing them with a new Literal op once.
- * For the moment, this only applies within a dag, later this should be
- * extended across statements block (global, inter-procedure).
- */
-public class RewriteConstantFolding extends HopRewriteRule
-{
-
- private static final String TMP_VARNAME = "__cf_tmp";
-
- //reuse basic execution runtime
- private static ProgramBlock _tmpPB = null;
- private static ExecutionContext _tmpEC = null;
-
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- if( roots == null )
- return null;
-
- for( int i=0; i<roots.size(); i++ )
- {
- Hop h = roots.get(i);
- roots.set(i, rule_ConstantFolding(h));
- }
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- if( root == null )
- return null;
-
- return rule_ConstantFolding(root);
- }
-
-
- /**
- *
- * @param hop
- * @throws HopsException
- */
- private Hop rule_ConstantFolding( Hop hop )
- throws HopsException
- {
- return rConstantFoldingExpression(hop);
- }
-
- /**
- *
- * @param root
- * @throws HopsException
- */
- private Hop rConstantFoldingExpression( Hop root )
- throws HopsException
- {
- if( root.getVisited() == VisitStatus.DONE )
- return root;
-
- //recursively process childs (before replacement to allow bottom-recursion)
- //no iterator in order to prevent concurrent modification
- for( int i=0; i<root.getInput().size(); i++ )
- {
- Hop h = root.getInput().get(i);
- rConstantFoldingExpression(h);
- }
-
- LiteralOp literal = null;
-
- //fold binary op if both are literals / unary op if literal
- if( root.getDataType() == DataType.SCALAR //scalar ouput
- && ( isApplicableBinaryOp(root) || isApplicableUnaryOp(root) ) )
- {
- //core constant folding via runtime instructions
- try {
- literal = evalScalarOperation(root);
- }
- catch(Exception ex)
- {
- LOG.error("Failed to execute constant folding instructions. No abort.", ex);
- }
-
- }
- //fold conjunctive predicate if at least one input is literal 'false'
- else if( isApplicableFalseConjunctivePredicate(root) )
- {
- literal = new LiteralOp(false);
- }
- //fold disjunctive predicate if at least one input is literal 'true'
- else if( isApplicableTrueDisjunctivePredicate(root) )
- {
- literal = new LiteralOp(true);
- }
-
- //replace binary operator with folded constant
- if( literal != null )
- {
- //reverse replacement in order to keep common subexpression elimination
- int plen = root.getParent().size();
- if( plen > 0 ) //broot is NOT a DAG root
- {
- for( int i=0; i<root.getParent().size(); i++ ) //for all parents
- {
- Hop parent = root.getParent().get(i);
- for( int j=0; j<parent.getInput().size(); j++ )
- {
- Hop child = parent.getInput().get(j);
- if( root == child )
- {
- //replace operator
- //root to parent link cannot be removed within this loop, as loop iterates over list containing parents.
- parent.getInput().remove(j);
- HopRewriteUtils.addChildReference(parent, literal,j);
- }
- }
- }
- root.getParent().clear();
- }
- else //broot IS a DAG root
- {
- root = literal;
- }
- }
-
-
- //mark processed
- root.setVisited( VisitStatus.DONE );
- return root;
- }
-
- /**
- * In order to (1) prevent unexpected side effects from constant folding and
- * (2) for simplicity with regard to arbitrary value type combinations,
- * we use the same compilation and runtime for constant folding as we would
- * use for actual instruction execution.
- *
- * @return
- * @throws IOException
- * @throws DMLUnsupportedOperationException
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws HopsException
- */
- private LiteralOp evalScalarOperation( Hop bop )
- throws LopsException, DMLRuntimeException, DMLUnsupportedOperationException, IOException, HopsException
- {
- //Timing time = new Timing( true );
-
- DataOp tmpWrite = new DataOp(TMP_VARNAME, bop.getDataType(), bop.getValueType(), bop, DataOpTypes.TRANSIENTWRITE, TMP_VARNAME);
-
- //generate runtime instruction
- Dag<Lop> dag = new Dag<Lop>();
- Recompiler.rClearLops(tmpWrite); //prevent lops reuse
- Lop lops = tmpWrite.constructLops(); //reconstruct lops
- lops.addToDag( dag );
- ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getConfig());
-
- //execute instructions
- ExecutionContext ec = getExecutionContext();
- ProgramBlock pb = getProgramBlock();
- pb.setInstructions( inst );
-
- pb.execute( ec );
-
- //get scalar result (check before invocation)
- ScalarObject so = (ScalarObject) ec.getVariable(TMP_VARNAME);
- LiteralOp literal = null;
- switch( bop.getValueType() ){
- case DOUBLE: literal = new LiteralOp(so.getDoubleValue()); break;
- case INT: literal = new LiteralOp(so.getLongValue()); break;
- case BOOLEAN: literal = new LiteralOp(so.getBooleanValue()); break;
- case STRING: literal = new LiteralOp(so.getStringValue()); break;
- default:
- throw new HopsException("Unsupported literal value type: "+bop.getValueType());
- }
-
- //cleanup
- tmpWrite.getInput().clear();
- bop.getParent().remove(tmpWrite);
- pb.setInstructions(null);
- ec.getVariables().removeAll();
-
- //set literal properties (scalar)
- literal.setDim1(0);
- literal.setDim2(0);
- literal.setRowsInBlock(-1);
- literal.setColsInBlock(-1);
-
- //System.out.println("Constant folded in "+time.stop()+"ms.");
-
- return literal;
- }
-
- /**
- *
- * @return
- * @throws DMLRuntimeException
- */
- private static ProgramBlock getProgramBlock()
- throws DMLRuntimeException
- {
- if( _tmpPB == null )
- _tmpPB = new ProgramBlock( new Program() );
- return _tmpPB;
- }
-
- /**
- *
- * @return
- */
- private static ExecutionContext getExecutionContext()
- {
- if( _tmpEC == null )
- _tmpEC = ExecutionContextFactory.createContext();
- return _tmpEC;
- }
-
- /**
- *
- * @param hop
- * @return
- */
- private boolean isApplicableBinaryOp( Hop hop )
- {
- ArrayList<Hop> in = hop.getInput();
- return ( hop instanceof BinaryOp
- && in.get(0) instanceof LiteralOp
- && in.get(1) instanceof LiteralOp
- && ((BinaryOp)hop).getOp()!=OpOp2.CBIND
- && ((BinaryOp)hop).getOp()!=OpOp2.RBIND);
-
- //string append is rejected although possible because it
- //messes up the explain runtime output due to introduced \n
- }
-
- /**
- *
- * @param hop
- * @return
- */
- private boolean isApplicableUnaryOp( Hop hop )
- {
- ArrayList<Hop> in = hop.getInput();
- return ( hop instanceof UnaryOp
- && in.get(0) instanceof LiteralOp
- && HopRewriteUtils.isValueTypeCast(((UnaryOp)hop).getOp()));
- }
-
- /**
- *
- * @param hop
- * @return
- * @throws HopsException
- */
- private boolean isApplicableFalseConjunctivePredicate( Hop hop )
- throws HopsException
- {
- ArrayList<Hop> in = hop.getInput();
- return ( hop instanceof BinaryOp
- && ((BinaryOp)hop).getOp()==OpOp2.AND
- && ( (in.get(0) instanceof LiteralOp && !((LiteralOp)in.get(0)).getBooleanValue())
- ||(in.get(1) instanceof LiteralOp && !((LiteralOp)in.get(1)).getBooleanValue())) );
- }
-
- /**
- *
- * @param hop
- * @return
- * @throws HopsException
- */
- private boolean isApplicableTrueDisjunctivePredicate( Hop hop )
- throws HopsException
- {
- ArrayList<Hop> in = hop.getInput();
- return ( hop instanceof BinaryOp
- && ((BinaryOp)hop).getOp()==OpOp2.OR
- && ( (in.get(0) instanceof LiteralOp && ((LiteralOp)in.get(0)).getBooleanValue())
- ||(in.get(1) instanceof LiteralOp && ((LiteralOp)in.get(1)).getBooleanValue())) );
- }
-}
[63/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
index 7a4eacb..75f0c9d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.ArrayList;
import java.util.HashSet;
@@ -30,38 +30,38 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.IDistributedCacheConsumer;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PickByCountInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixPackedCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRCombiner;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.GMRReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRBaseForCommonInstructions;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.IDistributedCacheConsumer;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.PickByCountInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixPackedCell;
+import org.apache.sysml.runtime.matrix.mapred.GMRCombiner;
+import org.apache.sysml.runtime.matrix.mapred.GMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.GMRReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRBaseForCommonInstructions;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.runtime.matrix.sort.PickFromCompactInputFormat;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
public class GMR
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
index bf5ec84..fdbd783 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -25,17 +25,17 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedInt;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.GroupedAggMRCombiner;
-import com.ibm.bi.dml.runtime.matrix.mapred.GroupedAggMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.GroupedAggMRReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedInt;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.mapred.GroupedAggMRCombiner;
+import org.apache.sysml.runtime.matrix.mapred.GroupedAggMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.GroupedAggMRReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class GroupedAggMR
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
index a19bea5..416274d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
@@ -16,11 +16,11 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
public class JobReturn
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java b/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
index 4446aca..a68b0c1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.HashSet;
@@ -27,24 +27,24 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMCJMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MMCJMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java b/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
index 8e88007..2277bd6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.HashSet;
@@ -26,20 +26,20 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.TripleIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMRJMRMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.MMRJMRReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.TaggedMatrixCell;
+import org.apache.sysml.runtime.matrix.data.TripleIndexes;
+import org.apache.sysml.runtime.matrix.mapred.MMRJMRMapper;
+import org.apache.sysml.runtime.matrix.mapred.MMRJMRReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.yarn.DMLAppMasterUtils;
/*
* inBlockRepresentation: indicate whether to use block representation or cell representation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
index 94f65a1..efb89b1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
@@ -16,52 +16,52 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.Serializable;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateUnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineTernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineUnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MMTSJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MapMultChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MatrixReshapeMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PMMJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ParameterizedBuiltinMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.QuaternaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RangeBasedReIndexInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReorgInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReplicateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ScalarInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UaggOuterChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryMRInstructionBase;
-import com.ibm.bi.dml.runtime.instructions.mr.ZeroOutInstruction;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateUnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineTernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineUnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CumulativeAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.MMTSJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MapMultChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.MatrixReshapeMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ParameterizedBuiltinMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.QuaternaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.instructions.mr.RangeBasedReIndexInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReorgInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReplicateInstruction;
+import org.apache.sysml.runtime.instructions.mr.ScalarInstruction;
+import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
+import org.apache.sysml.runtime.instructions.mr.TernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.UaggOuterChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryMRInstructionBase;
+import org.apache.sysml.runtime.instructions.mr.ZeroOutInstruction;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.IndexRange;
public class MatrixCharacteristics implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
index aefff6d..10aea7e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
public class MatrixDimensionsMetaData extends MetaData
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
index 3ecdc1a..1c8f299 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
@@ -16,10 +16,10 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
public class MatrixFormatMetaData extends MatrixDimensionsMetaData
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
index fd49fe3..4b12924 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
/**
* Class to store metadata associated with a file (e.g., a matrix) on disk.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
index f125425..4d65670 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.HashSet;
@@ -27,20 +27,20 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedAdaptivePartialBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.ReblockMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.ReblockReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedAdaptivePartialBlock;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.ReblockMapper;
+import org.apache.sysml.runtime.matrix.mapred.ReblockReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
/*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
index d992aed..b2afe1c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.io.IOException;
import java.net.URI;
@@ -42,33 +42,33 @@ import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.SortKeys;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryInstruction;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.runtime.matrix.sort.CompactInputFormat;
-import com.ibm.bi.dml.runtime.matrix.sort.CompactOutputFormat;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortComparable;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortComparableDesc;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortMapper;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortReducer;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortStitchupReducer;
-import com.ibm.bi.dml.runtime.matrix.sort.SamplingSortMRInputFormat;
-import com.ibm.bi.dml.runtime.matrix.sort.IndexSortStitchupMapper;
-import com.ibm.bi.dml.runtime.matrix.sort.ValueSortMapper;
-import com.ibm.bi.dml.runtime.matrix.sort.ValueSortReducer;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.SortKeys;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryInstruction;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.runtime.matrix.sort.CompactInputFormat;
+import org.apache.sysml.runtime.matrix.sort.CompactOutputFormat;
+import org.apache.sysml.runtime.matrix.sort.IndexSortComparable;
+import org.apache.sysml.runtime.matrix.sort.IndexSortComparableDesc;
+import org.apache.sysml.runtime.matrix.sort.IndexSortMapper;
+import org.apache.sysml.runtime.matrix.sort.IndexSortReducer;
+import org.apache.sysml.runtime.matrix.sort.IndexSortStitchupReducer;
+import org.apache.sysml.runtime.matrix.sort.SamplingSortMRInputFormat;
+import org.apache.sysml.runtime.matrix.sort.IndexSortStitchupMapper;
+import org.apache.sysml.runtime.matrix.sort.ValueSortMapper;
+import org.apache.sysml.runtime.matrix.sort.ValueSortReducer;
+import org.apache.sysml.runtime.util.MapReduceTool;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java b/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
index 2b57cb5..16d8d77 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix;
+package org.apache.sysml.runtime.matrix;
import java.util.HashMap;
@@ -26,21 +26,21 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters.Group;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVWriteMapper;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVWriteReducer;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
-import com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.MRInstructionParser;
+import org.apache.sysml.runtime.instructions.MRJobInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
+import org.apache.sysml.runtime.matrix.mapred.CSVWriteMapper;
+import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.ConvertTarget;
+import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
public class WriteCSVMR
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/AdaptivePartialBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/AdaptivePartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/AdaptivePartialBlock.java
index cd74c50..6ca0e72 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/AdaptivePartialBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/AdaptivePartialBlock.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java
index 9494c59..9cc134a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
index f658b8b..7971ac1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
@@ -16,9 +16,9 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
index 1ba5280..36327cb 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
@@ -16,10 +16,10 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class BinaryBlockToRowBlockConverter implements Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixBlock>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
index e271868..40aa86e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
@@ -16,13 +16,13 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java
index 9d33921..ec6cff9 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class BinaryCellToRowBlockConverter implements Converter<MatrixIndexes, MatrixCell, MatrixIndexes, MatrixBlock>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java
index 55c61df..110d388 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java
index 03aca1d..decbbb2 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -27,20 +27,20 @@ import java.util.Map;
import org.apache.hadoop.io.WritableComparable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.IndexRange;
@SuppressWarnings("rawtypes")
public class CM_N_COVCell extends MatrixValue implements WritableComparable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/CSVFileFormatProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/CSVFileFormatProperties.java b/src/main/java/org/apache/sysml/runtime/matrix/data/CSVFileFormatProperties.java
index c75f08d..7371032 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/CSVFileFormatProperties.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/CSVFileFormatProperties.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.Serializable;
-import com.ibm.bi.dml.parser.DataExpression;
+import org.apache.sysml.parser.DataExpression;
public class CSVFileFormatProperties extends FileFormatProperties implements Serializable
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/CTableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/CTableMap.java b/src/main/java/org/apache/sysml/runtime/matrix/data/CTableMap.java
index ab7b247..567973a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/CTableMap.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/CTableMap.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.util.LongLongDoubleHashMap;
-import com.ibm.bi.dml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
/**
* Ctable map is an abstraction for the hashmap used for ctable's hash group-by
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/Converter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/Converter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/Converter.java
index cfd3071..3bbccfd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/Converter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/Converter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/FileFormatProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FileFormatProperties.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FileFormatProperties.java
index d7fe446..0f4198f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FileFormatProperties.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FileFormatProperties.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class FileFormatProperties
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
index 3f6b4f3..e214b3c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
/**
* Helper class for external key/value exchange.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java
index 22ec8ca..b1a212e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/InputInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/InputInfo.java b/src/main/java/org/apache/sysml/runtime/matrix/data/InputInfo.java
index 26a3cd1..8460bee 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/InputInfo.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/InputInfo.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.Serializable;
@@ -30,9 +30,9 @@ import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-import com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.sort.PickFromCompactInputFormat;
@SuppressWarnings("rawtypes")
public class InputInfo implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
index 5a8a0da..ba6b306 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
import org.apache.commons.math3.linear.DecompositionSolver;
@@ -24,9 +24,9 @@ import org.apache.commons.math3.linear.LUDecomposition;
import org.apache.commons.math3.linear.QRDecomposition;
import org.apache.commons.math3.linear.RealMatrix;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.util.DataConverter;
/**
* Library for matrix operations that need invocation of
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
index 7881930..4afba68 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
import java.util.Arrays;
@@ -23,27 +23,27 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
-import com.ibm.bi.dml.runtime.functionobjects.IndexFunction;
-import com.ibm.bi.dml.runtime.functionobjects.KahanFunction;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlusSq;
-import com.ibm.bi.dml.runtime.functionobjects.Mean;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceDiag;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinFunctionCode;
+import org.apache.sysml.runtime.functionobjects.IndexFunction;
+import org.apache.sysml.runtime.functionobjects.KahanFunction;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
+import org.apache.sysml.runtime.functionobjects.Mean;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceDiag;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* MB:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
index 49a9687..3406711 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.Arrays;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.functionobjects.Divide;
-import com.ibm.bi.dml.runtime.functionobjects.Equals;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply2;
-import com.ibm.bi.dml.runtime.functionobjects.NotEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Or;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.Power2;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.SortUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.Divide;
+import org.apache.sysml.runtime.functionobjects.Equals;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Multiply2;
+import org.apache.sysml.runtime.functionobjects.NotEquals;
+import org.apache.sysml.runtime.functionobjects.Or;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.Power2;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.SortUtils;
/**
* MB:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
index 1a1ef96..337ad23 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,12 +29,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.random.Well1024a;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.NormalPRNGenerator;
-import com.ibm.bi.dml.runtime.util.PRNGenerator;
-import com.ibm.bi.dml.runtime.util.PoissonPRNGenerator;
-import com.ibm.bi.dml.runtime.util.UniformPRNGenerator;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.NormalPRNGenerator;
+import org.apache.sysml.runtime.util.PRNGenerator;
+import org.apache.sysml.runtime.util.PoissonPRNGenerator;
+import org.apache.sysml.runtime.util.UniformPRNGenerator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index 8196196..97e50c0 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,18 +26,18 @@ import java.util.concurrent.Executors;
import org.apache.commons.math3.util.FastMath;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy.WCeMMType;
-import com.ibm.bi.dml.lops.WeightedDivMM.WDivMMType;
-import com.ibm.bi.dml.lops.WeightedSigmoid.WSigmoidType;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss.WeightsType;
-import com.ibm.bi.dml.lops.WeightedUnaryMM.WUMMType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.lops.WeightedCrossEntropy.WCeMMType;
+import org.apache.sysml.lops.WeightedDivMM.WDivMMType;
+import org.apache.sysml.lops.WeightedSigmoid.WSigmoidType;
+import org.apache.sysml.lops.WeightedSquaredLoss.WeightsType;
+import org.apache.sysml.lops.WeightedUnaryMM.WUMMType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* MB:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
index c5dd83a..588bdef 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.Arrays;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Equals;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.NotEquals;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.SortUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Equals;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.NotEquals;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.SortUtils;
/**
* ACS:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index 31fe405..9060d97 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,16 +26,16 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SortIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.SortUtils;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.SortIndex;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.SortUtils;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* MB:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 18fce81..9686932 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -34,55 +34,55 @@ import java.util.Map;
import org.apache.commons.math3.random.Well1024a;
import org.apache.hadoop.io.DataInputBuffer;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.MMTSJ.MMTSJType;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.functionobjects.CTable;
-import com.ibm.bi.dml.runtime.functionobjects.DiagIndex;
-import com.ibm.bi.dml.runtime.functionobjects.Divide;
-import com.ibm.bi.dml.runtime.functionobjects.KahanFunction;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlusSq;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.functionobjects.SortIndex;
-import com.ibm.bi.dml.runtime.functionobjects.SwapIndex;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixBincell.BinaryAccessType;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataInputStream;
-import com.ibm.bi.dml.runtime.util.FastBufferedDataOutputStream;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.functionobjects.CTable;
+import org.apache.sysml.runtime.functionobjects.DiagIndex;
+import org.apache.sysml.runtime.functionobjects.Divide;
+import org.apache.sysml.runtime.functionobjects.KahanFunction;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.functionobjects.SortIndex;
+import org.apache.sysml.runtime.functionobjects.SwapIndex;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixBincell.BinaryAccessType;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class MatrixBlock extends MatrixValue implements Externalizable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataInput.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataInput.java
index 838eb8c..08b23d3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataInput.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataInput.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataOutput.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataOutput.java
index f79bc57..4f3ee40 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataOutput.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlockDataOutput.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java
index 0fb2be8..5b9d7f2 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -28,21 +28,21 @@ import java.util.Map;
import org.apache.hadoop.io.WritableComparable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.CTable;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceDiag;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.CTable;
+import org.apache.sysml.runtime.functionobjects.ReduceDiag;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.IndexRange;
@SuppressWarnings("rawtypes")
public class MatrixCell extends MatrixValue implements WritableComparable, Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
index 8f5e6c2..9cb6d1d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
* This represent the indexes to the blocks of the matrix.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java
index dd1bad1..ad29fbe 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java
@@ -16,17 +16,17 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.cp.KahanObject;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.cp.KahanObject;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
public class MatrixPackedCell extends MatrixCell
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java
index 7afbcec..ac1aaad 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
import java.util.Collection;
@@ -25,20 +25,20 @@ import java.util.Map;
import org.apache.hadoop.io.WritableComparable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.QuaternaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
@SuppressWarnings("rawtypes")
public abstract class MatrixValue implements WritableComparable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/MultipleOutputCommitter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MultipleOutputCommitter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MultipleOutputCommitter.java
index b9a3e6a..c293799 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MultipleOutputCommitter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MultipleOutputCommitter.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.IOException;
@@ -30,7 +30,7 @@ import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.TaskAttemptContext;
import org.apache.hadoop.mapred.TaskAttemptID;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
public class MultipleOutputCommitter extends FileOutputCommitter
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
index 1b1d9e2..ec2c246 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
@@ -16,10 +16,10 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
public class NumItemsByEachReducerMetaData extends MatrixDimensionsMetaData
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index fd10ba2..01e4762 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -16,25 +16,25 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class OperationsOnMatrixValues
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
index 01365f1..c08d03d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.Serializable;
@@ -29,9 +29,9 @@ import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput;
-import com.ibm.bi.dml.runtime.matrix.sort.CompactOutputFormat;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput;
+import org.apache.sysml.runtime.matrix.sort.CompactOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/Pair.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/Pair.java b/src/main/java/org/apache/sysml/runtime/matrix/data/Pair.java
index f8de328..07bf6c0 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/Pair.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/Pair.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
public class Pair<K, V>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java
index 7ac8ed6..f32d8fc 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java
@@ -16,7 +16,7 @@
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java b/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
index f42c804..6742e05 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
@@ -15,10 +15,10 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.PoissonPRNGenerator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.PoissonPRNGenerator;
public class PoissonRandomMatrixGenerator extends RandomMatrixGenerator {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/matrix/data/RandomMatrixGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/RandomMatrixGenerator.java b/src/main/java/org/apache/sysml/runtime/matrix/data/RandomMatrixGenerator.java
index e31515e..11213d6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/RandomMatrixGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/RandomMatrixGenerator.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.matrix.data;
+package org.apache.sysml.runtime.matrix.data;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.NormalPRNGenerator;
-import com.ibm.bi.dml.runtime.util.PRNGenerator;
-import com.ibm.bi.dml.runtime.util.PoissonPRNGenerator;
-import com.ibm.bi.dml.runtime.util.UniformPRNGenerator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.NormalPRNGenerator;
+import org.apache.sysml.runtime.util.PRNGenerator;
+import org.apache.sysml.runtime.util.PoissonPRNGenerator;
+import org.apache.sysml.runtime.util.UniformPRNGenerator;
public class RandomMatrixGenerator {
[66/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGAlignedSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGAlignedSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGAlignedSPInstruction.java
index 67dc07e..3d7ed33 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGAlignedSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGAlignedSPInstruction.java
@@ -15,26 +15,26 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.OffsetColumnIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class AppendGAlignedSPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java
index d86e2f5..16be42a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.Iterator;
@@ -26,20 +26,20 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.OffsetColumnIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class AppendGSPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendMSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendMSPInstruction.java
index 76f198c..c71cf55 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendMSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendMSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.Iterator;
@@ -25,24 +25,24 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.OffsetColumnIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class AppendMSPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendRSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendRSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendRSPInstruction.java
index bff74d8..577692c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendRSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendRSPInstruction.java
@@ -15,25 +15,25 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.OffsetColumnIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class AppendRSPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ArithmeticBinarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ArithmeticBinarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ArithmeticBinarySPInstruction.java
index bac5bb0..f6b6817 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ArithmeticBinarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ArithmeticBinarySPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class ArithmeticBinarySPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinUaggChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BinUaggChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinUaggChainSPInstruction.java
index 1f0aabc..859a06a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/BinUaggChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinUaggChainSPInstruction.java
@@ -15,22 +15,22 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
public class BinUaggChainSPInstruction extends UnarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BinarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinarySPInstruction.java
index 0736b65..c536994 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/BinarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BinarySPInstruction.java
@@ -15,31 +15,31 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.MatrixMatrixBinaryOpFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.MatrixScalarUnaryFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.MatrixVectorBinaryOpPartitionFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.OuterVectorBinaryOpFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ReplicateVectorFunction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.MatrixMatrixBinaryOpFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.MatrixScalarUnaryFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.MatrixVectorBinaryOpPartitionFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.OuterVectorBinaryOpFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.ReplicateVectorFunction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
public abstract class BinarySPInstruction extends ComputationSPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinBinarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinBinarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinBinarySPInstruction.java
index 3ec50d6..a5c9944 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinBinarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinBinarySPInstruction.java
@@ -1,19 +1,19 @@
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.RightScalarOperator;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.RightScalarOperator;
public abstract class BuiltinBinarySPInstruction extends BinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinUnarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinUnarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinUnarySPInstruction.java
index baf3608..51140ff 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinUnarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinUnarySPInstruction.java
@@ -1,15 +1,15 @@
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public abstract class BuiltinUnarySPInstruction extends UnarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index 479bdb6..ab58541 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDConverterUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CSVReblockSPInstruction extends UnarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java
index 34aecfa..e79e582 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
@@ -23,23 +23,23 @@ import org.apache.spark.api.java.function.Function2;
import scala.Tuple2;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
index affb0d5..68857a6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
@@ -15,28 +15,28 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.storage.StorageLevel;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.CopyBlockFunction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockFunction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CheckpointSPInstruction extends UnarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ComputationSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ComputationSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ComputationSPInstruction.java
index 934afd4..def2a8d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ComputationSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ComputationSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class ComputationSPInstruction extends SPInstruction {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CovarianceSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CovarianceSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CovarianceSPInstruction.java
index 393474f..5503385 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CovarianceSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CovarianceSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
@@ -23,21 +23,21 @@ import org.apache.spark.api.java.function.Function2;
import scala.Tuple2;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CM_COV_Object;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.DoubleObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java
index 4e65c66..78c5257 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java
@@ -15,29 +15,29 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
* Cpmm: cross-product matrix multiplication operation (distributed matrix multiply
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java
index b9be32f..ca2a407 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
@@ -23,18 +23,18 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java
index 722a8b0..1bb2db6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -25,22 +25,22 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public class CumulativeOffsetSPInstruction extends BinarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
index 637a6c0..191c8f8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
@@ -24,19 +24,19 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.MapMultChain.ChainType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.MapMultChain.ChainType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
index f9e0acc..57e5009 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -27,28 +27,28 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMult.CacheType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.FilterNonEmptyBlocksFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.MapMult.CacheType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.FilterNonEmptyBlocksFunction;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
* TODO: we need to reason about multiple broadcast variables for chains of mapmults (sum of operations until cleanup)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorArithmeticSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorArithmeticSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorArithmeticSPInstruction.java
index 5690344..e5aca4b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorArithmeticSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorArithmeticSPInstruction.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixBVectorArithmeticSPInstruction extends ArithmeticBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorBuiltinSPInstruction.java
index d9ac52a..b172994 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorBuiltinSPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixBVectorBuiltinSPInstruction extends BuiltinBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorRelationalSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorRelationalSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorRelationalSPInstruction.java
index b80aade..30e8cc4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorRelationalSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBVectorRelationalSPInstruction.java
@@ -16,14 +16,14 @@
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.lops.BinaryM.VectorType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.BinaryM.VectorType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixBVectorRelationalSPInstruction extends RelationalBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBuiltinSPInstruction.java
index c6c50a1..26ab9a2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixBuiltinSPInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index 08b5c5e..afbe362 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.Iterator;
@@ -26,28 +26,28 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LazyIterableIterator;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.IsBlockInRange;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.util.IndexRange;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.IsBlockInRange;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class MatrixIndexingSPInstruction extends UnarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixArithmeticSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixArithmeticSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixArithmeticSPInstruction.java
index 67d89fb..b8a3ac4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixArithmeticSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixArithmeticSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixArithmeticSPInstruction extends ArithmeticBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixBuiltinSPInstruction.java
index 6310801..ea06341 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixBuiltinSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixBuiltinSPInstruction extends BuiltinBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixRelationalSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixRelationalSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixRelationalSPInstruction.java
index 248bb6d..abeba24 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixRelationalSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixMatrixRelationalSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixMatrixRelationalSPInstruction extends RelationalBinarySPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java
index 2aabad0..8f68832 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -24,22 +24,22 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarArithmeticSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarArithmeticSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarArithmeticSPInstruction.java
index 4f3046c..ddeff1a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarArithmeticSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarArithmeticSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixScalarArithmeticSPInstruction extends ArithmeticBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
index ff77540..2e47801 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixScalarBuiltinSPInstruction extends BuiltinBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarRelationalSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarRelationalSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarRelationalSPInstruction.java
index c417789..241101b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarRelationalSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarRelationalSPInstruction.java
@@ -1,10 +1,10 @@
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class MatrixScalarRelationalSPInstruction extends RelationalBinarySPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java
index b571d62..94013b8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -28,25 +28,25 @@ import org.apache.spark.storage.StorageLevel;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.PMapMult;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedMatrixBlock;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.IsBlockInRange;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OperationsOnMatrixValues;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.lops.PMapMult;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.functions.IsBlockInRange;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
/**
* This pmapmm matrix multiplication instruction is still experimental
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
index 439af56..f277d3f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,43 +27,43 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ExtractGroup;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ExtractGroupNWeights;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.PerformGroupByAggInCombiner;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.PerformGroupByAggInReducer;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.ReplicateVectorFunction;
-import com.ibm.bi.dml.runtime.instructions.spark.functions.UnflattenIterablesAfterCogroup;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.SparkUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixReorg;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.WeightedCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.IndexedMatrixValue;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.transform.DataTransform;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.ParameterizedBuiltin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.ExtractGroup;
+import org.apache.sysml.runtime.instructions.spark.functions.ExtractGroupNWeights;
+import org.apache.sysml.runtime.instructions.spark.functions.PerformGroupByAggInCombiner;
+import org.apache.sysml.runtime.instructions.spark.functions.PerformGroupByAggInReducer;
+import org.apache.sysml.runtime.instructions.spark.functions.ReplicateVectorFunction;
+import org.apache.sysml.runtime.instructions.spark.functions.UnflattenIterablesAfterCogroup;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.WeightedCell;
+import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.transform.DataTransform;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java
index 1a6b2fb..5e291eb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.spark;
+package org.apache.sysml.runtime.instructions.spark;
import java.util.ArrayList;
@@ -25,26 +25,26 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.MapMult.CacheType;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.instructions.cp.CPOperand;
-import com.ibm.bi.dml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.MapMult.CacheType;
+import org.apache.sysml.lops.PMMJ;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
*
[69/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Not.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Not.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Not.java
index e045847..83404c7 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Not.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Not.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class Not extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/NotEquals.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/NotEquals.java b/src/main/java/org/apache/sysml/runtime/functionobjects/NotEquals.java
index e6a74e3..b0818de 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/NotEquals.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/NotEquals.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
public class NotEquals extends ValueFunction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
index 41bfbd4..ac81920 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class OffsetColumnIndex extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Or.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Or.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Or.java
index 61c3c63..11b34a8 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Or.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Or.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java
index 0734b61..ed83fd4 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.util.HashMap;
@@ -27,8 +27,8 @@ import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.commons.math3.distribution.TDistribution;
import org.apache.commons.math3.exception.MathArithmeticException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.UtilFunctions;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Plus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Plus.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Plus.java
index c185639..bc89c24 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Plus.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Plus.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLRuntimeException;
// Singleton class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Power.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Power.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Power.java
index 638ff0d..5bd404b 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Power.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Power.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
// Singleton class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/Power2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Power2.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Power2.java
index 87d91b8..5b9c2aa 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Power2.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Power2.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
// Singleton class
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceAll.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceAll.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceAll.java
index 574cfd3..11ddeeb 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceAll.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceAll.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class ReduceAll extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceCol.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceCol.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceCol.java
index 348a44c..70c242d 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceCol.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceCol.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class ReduceCol extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceDiag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceDiag.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceDiag.java
index f7c9592..479b833 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceDiag.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceDiag.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class ReduceDiag extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceRow.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceRow.java
index 8c08e0a..04970a0 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceRow.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ReduceRow.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class ReduceRow extends IndexFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/RemoveFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/RemoveFile.java b/src/main/java/org/apache/sysml/runtime/functionobjects/RemoveFile.java
index 2a6bc73..a39aaf8 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/RemoveFile.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/RemoveFile.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class RemoveFile extends FileFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/RenameFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/RenameFile.java b/src/main/java/org/apache/sysml/runtime/functionobjects/RenameFile.java
index a865306..8cc05a4 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/RenameFile.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/RenameFile.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.IOException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.util.MapReduceTool;
public class RenameFile extends FileFunction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/SortIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/SortIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/SortIndex.java
index e045513..c730693 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/SortIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/SortIndex.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
/**
* This index function is NOT used for actual sorting but just as a reference
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/SwapIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/SwapIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/SwapIndex.java
index 5b5906b..4b76132 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/SwapIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/SwapIndex.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
public class SwapIndex extends IndexFunction implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/functionobjects/ValueFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ValueFunction.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ValueFunction.java
index af42955..2940aba 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/ValueFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ValueFunction.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.runtime.functionobjects;
+package org.apache.sysml.runtime.functionobjects;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index c9744a9..67ca43c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -15,50 +15,50 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.UnaryCP;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateTernaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AggregateUnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.AppendCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanUnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BuiltinBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BuiltinUnaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CentralMomentCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CovarianceCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.DataGenCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.DataPartitionCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FileCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.FunctionCallCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MMChainCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MMTSJCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MatrixIndexingCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MatrixReshapeCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.PMMJCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.QuantilePickCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.QuantileSortCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.QuaternaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.RelationalBinaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ReorgCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.StringInitCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.TernaryCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.UaggOuterChainCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cpfile.MatrixIndexingCPFileInstruction;
-import com.ibm.bi.dml.runtime.instructions.cpfile.ParameterizedBuiltinCPFileInstruction;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.cp.AggregateBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.AggregateTernaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.AggregateUnaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BooleanUnaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BuiltinBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BuiltinUnaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CentralMomentCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CovarianceCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.DataPartitionCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.FileCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MMChainCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MMTSJCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MatrixIndexingCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MatrixReshapeCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.PMMJCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.QuantilePickCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.QuantileSortCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.QuaternaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.RelationalBinaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.ReorgCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.StringInitCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.TernaryCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.UaggOuterChainCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.cpfile.MatrixIndexingCPFileInstruction;
+import org.apache.sysml.runtime.instructions.cpfile.ParameterizedBuiltinCPFileInstruction;
public class CPInstructionParser extends InstructionParser
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/Instruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/Instruction.java b/src/main/java/org/apache/sysml/runtime/instructions/Instruction.java
index 68ad8e7..1f563f6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/Instruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/Instruction.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.ibm.bi.dml.api.monitoring.Location;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.api.monitoring.Location;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
public abstract class Instruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/InstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionParser.java
index 01e2688..8e38d20 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionParser.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
public class InstructionParser
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
index d32ca61..4c4a4d9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
@@ -15,65 +15,65 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import java.util.StringTokenizer;
-import com.ibm.bi.dml.lops.AppendM;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.PMMJ;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.UAggOuterChain;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSigmoidR;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.And;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.Divide;
-import com.ibm.bi.dml.runtime.functionobjects.Equals;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThan;
-import com.ibm.bi.dml.runtime.functionobjects.GreaterThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.IntegerDivide;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlusSq;
-import com.ibm.bi.dml.runtime.functionobjects.LessThan;
-import com.ibm.bi.dml.runtime.functionobjects.LessThanEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Mean;
-import com.ibm.bi.dml.runtime.functionobjects.Minus;
-import com.ibm.bi.dml.runtime.functionobjects.Minus1Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.MinusNz;
-import com.ibm.bi.dml.runtime.functionobjects.Modulus;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply2;
-import com.ibm.bi.dml.runtime.functionobjects.NotEquals;
-import com.ibm.bi.dml.runtime.functionobjects.Or;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.functionobjects.Power;
-import com.ibm.bi.dml.runtime.functionobjects.Power2;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceAll;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceCol;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceDiag;
-import com.ibm.bi.dml.runtime.functionobjects.ReduceRow;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.LeftScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.RightScalarOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.lops.AppendM;
+import org.apache.sysml.lops.BinaryM;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.PMMJ;
+import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
+import org.apache.sysml.lops.UAggOuterChain;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedCrossEntropyR;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.lops.WeightedSigmoidR;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.And;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Divide;
+import org.apache.sysml.runtime.functionobjects.Equals;
+import org.apache.sysml.runtime.functionobjects.GreaterThan;
+import org.apache.sysml.runtime.functionobjects.GreaterThanEquals;
+import org.apache.sysml.runtime.functionobjects.IntegerDivide;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
+import org.apache.sysml.runtime.functionobjects.LessThan;
+import org.apache.sysml.runtime.functionobjects.LessThanEquals;
+import org.apache.sysml.runtime.functionobjects.Mean;
+import org.apache.sysml.runtime.functionobjects.Minus;
+import org.apache.sysml.runtime.functionobjects.Minus1Multiply;
+import org.apache.sysml.runtime.functionobjects.MinusNz;
+import org.apache.sysml.runtime.functionobjects.Modulus;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Multiply2;
+import org.apache.sysml.runtime.functionobjects.NotEquals;
+import org.apache.sysml.runtime.functionobjects.Or;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.functionobjects.Power;
+import org.apache.sysml.runtime.functionobjects.Power2;
+import org.apache.sysml.runtime.functionobjects.ReduceAll;
+import org.apache.sysml.runtime.functionobjects.ReduceCol;
+import org.apache.sysml.runtime.functionobjects.ReduceDiag;
+import org.apache.sysml.runtime.functionobjects.ReduceRow;
+import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.LeftScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.RightScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
public class InstructionUtils
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
index 932cb08..1efb8af 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
@@ -15,67 +15,67 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.BinaryM;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.MapMult;
-import com.ibm.bi.dml.lops.MapMultChain;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSigmoidR;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AggregateUnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.AppendInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinUaggChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.BinaryMInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CM_N_COVInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CSVWriteInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineBinaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineTernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CombineUnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeOffsetInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.CumulativeSplitInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.DataPartitionMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MMTSJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MapMultChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MatrixReshapeMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PMMJMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ParameterizedBuiltinMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.PickByCountInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.QuaternaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RangeBasedReIndexInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RemoveEmptyMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReorgInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReplicateInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ScalarInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.TernaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UaggOuterChainInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.UnaryInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ZeroOutInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
+import org.apache.sysml.lops.BinaryM;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedCrossEntropyR;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.lops.WeightedSigmoidR;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.AggregateUnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.AppendInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinUaggChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.BinaryMInstruction;
+import org.apache.sysml.runtime.instructions.mr.CM_N_COVInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineBinaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineTernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CombineUnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.CumulativeAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.CumulativeOffsetInstruction;
+import org.apache.sysml.runtime.instructions.mr.CumulativeSplitInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.DataPartitionMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction;
+import org.apache.sysml.runtime.instructions.mr.MMTSJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction;
+import org.apache.sysml.runtime.instructions.mr.MapMultChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.MatrixReshapeMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ParameterizedBuiltinMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.PickByCountInstruction;
+import org.apache.sysml.runtime.instructions.mr.QuaternaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.RandInstruction;
+import org.apache.sysml.runtime.instructions.mr.RangeBasedReIndexInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReblockInstruction;
+import org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReorgInstruction;
+import org.apache.sysml.runtime.instructions.mr.ReplicateInstruction;
+import org.apache.sysml.runtime.instructions.mr.ScalarInstruction;
+import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
+import org.apache.sysml.runtime.instructions.mr.TernaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.UaggOuterChainInstruction;
+import org.apache.sysml.runtime.instructions.mr.UnaryInstruction;
+import org.apache.sysml.runtime.instructions.mr.ZeroOutInstruction;
+import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.matrix.SortMR;
public class MRInstructionParser extends InstructionParser
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
index 8c7883a..9d57af2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
@@ -15,37 +15,37 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
+import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.JobReturn;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
+import org.apache.sysml.runtime.util.UtilFunctions;
+import org.apache.sysml.utils.Statistics;
/*
---------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index 62c49b8..bd4d9d0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -15,63 +15,63 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions;
+package org.apache.sysml.runtime.instructions;
import java.util.HashMap;
-import com.ibm.bi.dml.lops.Checkpoint;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.WeightedCrossEntropy;
-import com.ibm.bi.dml.lops.WeightedCrossEntropyR;
-import com.ibm.bi.dml.lops.WeightedDivMM;
-import com.ibm.bi.dml.lops.WeightedDivMMR;
-import com.ibm.bi.dml.lops.WeightedSigmoid;
-import com.ibm.bi.dml.lops.WeightedSigmoidR;
-import com.ibm.bi.dml.lops.WeightedSquaredLoss;
-import com.ibm.bi.dml.lops.WeightedSquaredLossR;
-import com.ibm.bi.dml.lops.WeightedUnaryMM;
-import com.ibm.bi.dml.lops.WeightedUnaryMMR;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.spark.AggregateTernarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.AggregateUnarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.AppendGAlignedSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.AppendGSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.AppendMSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.AppendRSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ArithmeticBinarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.BinUaggChainSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.BuiltinBinarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.BuiltinUnarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CSVReblockSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CentralMomentSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CheckpointSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CovarianceSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CpmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CumulativeAggregateSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.CumulativeOffsetSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.MapmmChainSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.MapmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.MatrixIndexingSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.MatrixReshapeSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.PMapmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.PmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.QuantilePickSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.QuaternarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.RandSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ReblockSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.RelationalBinarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ReorgSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.RmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.spark.TernarySPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.TsmmSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.QuantileSortSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.UaggOuterChainSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.WriteSPInstruction;
-import com.ibm.bi.dml.runtime.instructions.spark.ZipmmSPInstruction;
+import org.apache.sysml.lops.Checkpoint;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.WeightedCrossEntropy;
+import org.apache.sysml.lops.WeightedCrossEntropyR;
+import org.apache.sysml.lops.WeightedDivMM;
+import org.apache.sysml.lops.WeightedDivMMR;
+import org.apache.sysml.lops.WeightedSigmoid;
+import org.apache.sysml.lops.WeightedSigmoidR;
+import org.apache.sysml.lops.WeightedSquaredLoss;
+import org.apache.sysml.lops.WeightedSquaredLossR;
+import org.apache.sysml.lops.WeightedUnaryMM;
+import org.apache.sysml.lops.WeightedUnaryMMR;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.spark.AggregateTernarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.AggregateUnarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.AppendGAlignedSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.AppendMSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.AppendRSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ArithmeticBinarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.BinUaggChainSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.BuiltinBinarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.BuiltinUnarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CSVReblockSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CentralMomentSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CheckpointSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CovarianceSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CpmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CumulativeAggregateSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CumulativeOffsetSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.MapmmChainSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.MapmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.MatrixIndexingSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.MatrixReshapeSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.PMapmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.PmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.QuantilePickSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.QuaternarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.RandSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ReblockSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.RelationalBinarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ReorgSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.RmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction;
+import org.apache.sysml.runtime.instructions.spark.SPInstruction.SPINSTRUCTION_TYPE;
+import org.apache.sysml.runtime.instructions.spark.TernarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.TsmmSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.QuantileSortSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.UaggOuterChainSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.WriteSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.ZipmmSPInstruction;
public class SPInstructionParser extends InstructionParser {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java
index 19776fa..de17f74 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java
@@ -15,20 +15,20 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.functionobjects.Plus;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.functionobjects.Plus;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AggregateBinaryCPInstruction extends BinaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateTernaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateTernaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateTernaryCPInstruction.java
index 4708541..cfb2384 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateTernaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateTernaryCPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.KahanPlus;
-import com.ibm.bi.dml.runtime.functionobjects.Multiply;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateBinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.Multiply;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class AggregateTernaryCPInstruction extends ComputationCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
index 63d1815..0e78c2b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
@@ -15,24 +15,24 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.operators.AggregateUnaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public class AggregateUnaryCPInstruction extends UnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
index a056daa..f492750 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.OffsetColumnIndex;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.ReorgOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
public class AppendCPInstruction extends BinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ArithmeticBinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ArithmeticBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ArithmeticBinaryCPInstruction.java
index 672a33a..a21c076 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ArithmeticBinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ArithmeticBinaryCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class ArithmeticBinaryCPInstruction extends BinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BinaryCPInstruction.java
index def535e..8b63274 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BinaryCPInstruction.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class BinaryCPInstruction extends ComputationCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanBinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanBinaryCPInstruction.java
index 30452e2..afbe6f6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanBinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanBinaryCPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class BooleanBinaryCPInstruction extends BinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
index cd0db26..809cdb9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
@@ -15,9 +15,9 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
+import org.apache.sysml.parser.Expression.ValueType;
public class BooleanObject extends ScalarObject
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanUnaryCPInstruction.java
index 5ea9587..d584fe4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanUnaryCPInstruction.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public class BooleanUnaryCPInstruction extends UnaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BreakPointInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BreakPointInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BreakPointInstruction.java
index fc2d581..e5fb6a5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BreakPointInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BreakPointInstruction.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DebugState;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.debug.DebugState;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinBinaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinBinaryCPInstruction.java
index c02533b..c044d41 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinBinaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinBinaryCPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.operators.BinaryOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.RightScalarOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.RightScalarOperator;
public abstract class BuiltinBinaryCPInstruction extends BinaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
index 8a56450..edae21f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
@@ -15,19 +15,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.functionobjects.Builtin;
-import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.UnaryOperator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/CM_COV_Object.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CM_COV_Object.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CM_COV_Object.java
index 10bdc57..816c71d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CM_COV_Object.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CM_COV_Object.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
public class CM_COV_Object extends Data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
index 66d78f4..2585899 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.api.MLContextProxy;
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.lops.runtime.RunMRJobs;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.CPInstructionParser;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class CPInstruction extends Instruction
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
index 3341b5e..990013c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.*;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.parser.Expression.*;
+import org.apache.sysml.runtime.instructions.Instruction;
public class CPOperand
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
index 34d3361..b72a6bf 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
@@ -16,19 +16,19 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.CM;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
public class CentralMomentCPInstruction extends AggregateUnaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
index ddd49ac..67d15a8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
@@ -15,11 +15,11 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public abstract class ComputationCPInstruction extends CPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/CovarianceCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CovarianceCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CovarianceCPInstruction.java
index 49129a3..433d334 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CovarianceCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CovarianceCPInstruction.java
@@ -15,18 +15,18 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.functionobjects.COV;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.operators.COVOperator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.functionobjects.COV;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.COVOperator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
public class CovarianceCPInstruction extends BinaryCPInstruction
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/Data.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/Data.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/Data.java
index 07e7d0d..4047f3f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/Data.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/Data.java
@@ -15,15 +15,15 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
import java.io.Serializable;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MetaData;
public abstract class Data implements Serializable
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
index d97a4d4..a928e23 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
@@ -15,21 +15,21 @@
*
*/
-package com.ibm.bi.dml.runtime.instructions.cp;
+package org.apache.sysml.runtime.instructions.cp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
-import com.ibm.bi.dml.runtime.matrix.operators.Operator;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
+import org.apache.sysml.hops.DataGenOp;
+import org.apache.sysml.hops.Hop.DataGenMethod;
+import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibMatrixDatagen;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.RandomMatrixGenerator;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.UtilFunctions;
public class DataGenCPInstruction extends UnaryCPInstruction
{
[03/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
deleted file mode 100644
index 9d6f320..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.StatisticMonitor;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * Remote ParWorker implementation, realized as MR mapper.
- *
- * NOTE: In a cluster setup, reuse jvm will not lead to reusing jvms of different jobs or different
- * task types due to job-level specification of jvm max sizes for map/reduce
- *
- */
-public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not required (no op implementations of configure, close)
- implements Mapper<LongWritable, Text, Writable, Writable>
-{
-
- //cache for future reuse (in case of JVM reuse)
- //NOTE: Hashmap to support multiple parfor MR jobs for local mode and if JVM reuse across jobs
- private static HashMap<String,RemoteParWorkerMapper> _sCache = null;
-
- //MR ParWorker attributes
- protected String _stringID = null;
- protected HashMap<String, String> _rvarFnames = null;
-
- static
- {
- //init cache (once per JVM)
- _sCache = new HashMap<String, RemoteParWorkerMapper>();
- }
-
-
- public RemoteParWorkerMapper( )
- {
- //only used if JVM reuse is enabled in order to ensure consistent output
- //filenames across tasks of one reused worker (preaggregation)
- _rvarFnames = new HashMap<String, String>();
- }
-
- /**
- *
- */
- @Override
- public void map(LongWritable key, Text value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- LOG.trace("execute RemoteParWorkerMapper "+_stringID+" ("+_workerID+")");
-
- //state for jvm reuse and multiple iterations
- long numIters = getExecutedIterations();
-
- try
- {
- //parse input task
- Task lTask = Task.parseCompactString( value.toString() );
-
- //execute task (on error: re-try via Hadoop)
- executeTask( lTask );
-
- //write output if required (matrix indexed write)
- RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars, _rvarFnames, out );
- }
- catch(Exception ex)
- {
- //throw IO exception to adhere to API specification
- throw new IOException("ParFOR: Failed to execute task.",ex);
- }
-
- //statistic maintenance
- RemoteParForUtils.incrementParForMRCounters(reporter, 1, getExecutedIterations()-numIters);
-
- //print heaver hitter per task
- JobConf job = ConfigurationManager.getCachedJobConf();
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- LOG.info("\nSystemML Statistics:\nHeavy hitter instructions (name, time, count):\n" + Statistics.getHeavyHitters(10));
- }
-
- /**
- *
- */
- @Override
- public void configure(JobConf job)
- {
- boolean requiresConfigure = true;
- String jobID = job.get("mapred.job.id");
-
- //probe cache for existing worker (parfor body, symbol table, etc)
- if( ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER )
- {
- synchronized( _sCache ) //for multiple jobs in local mode
- {
- if( _sCache.containsKey(jobID) )
- {
- RemoteParWorkerMapper tmp = _sCache.get(jobID);
-
- _stringID = tmp._stringID;
- _workerID = tmp._workerID;
-
- _childBlocks = tmp._childBlocks;
- _resultVars = tmp._resultVars;
- _ec = tmp._ec;
-
- _numIters = tmp._numIters;
- _numTasks = tmp._numTasks;
-
- _rvarFnames = tmp._rvarFnames;
-
- requiresConfigure = false;
- }
- }
- }
-
- if( requiresConfigure )
- {
- LOG.trace("configure RemoteParWorkerMapper "+job.get("mapred.tip.id"));
-
- try
- {
- _stringID = job.get("mapred.tip.id"); //task ID
- _workerID = IDHandler.extractIntID(_stringID); //int task ID
-
- //use the given job configuration as source for all new job confs
- //NOTE: this is required because on HDP 2.3, the classpath of mr tasks contained hadoop-common.jar
- //which includes a core-default.xml configuration which hides the actual default cluster configuration
- //in the context of mr jobs (for example this config points to local fs instead of hdfs by default).
- if( !InfrastructureAnalyzer.isLocalMode(job) ) {
- ConfigurationManager.setCachedJobConf(job);
- }
-
- //create local runtime program
- String in = MRJobConfiguration.getProgramBlocks(job);
- ParForBody body = ProgramConverter.parseParForBody(in, (int)_workerID);
- _childBlocks = body.getChildBlocks();
- _ec = body.getEc();
- _resultVars = body.getResultVarNames();
-
- //init local cache manager
- if( !CacheableData.isCachingActive() ) {
- String uuid = IDHandler.createDistributedUniqueID();
- LocalFileUtils.createWorkingDirectoryWithUUID( uuid );
- CacheableData.initCaching( uuid ); //incl activation, cache dir creation (each map task gets its own dir for simplified cleanup)
- }
- if( !CacheableData.cacheEvictionLocalFilePrefix.contains("_") ){ //account for local mode
- CacheableData.cacheEvictionLocalFilePrefix = CacheableData.cacheEvictionLocalFilePrefix +"_" + _workerID;
- }
-
- //ensure that resultvar files are not removed
- super.pinResultVariables();
-
- //enable/disable caching (if required)
- boolean cpCaching = MRJobConfiguration.getParforCachingConfig( job );
- if( !cpCaching )
- CacheableData.disableCaching();
-
- _numTasks = 0;
- _numIters = 0;
-
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
-
- //disable stat monitoring, reporting execution times via counters not useful
- StatisticMonitor.disableStatMonitoring();
-
- //put into cache if required
- if( ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER )
- synchronized( _sCache ){ //for multiple jobs in local mode
- _sCache.put(jobID, this);
- }
- }
- else
- {
- LOG.trace("reuse configured RemoteParWorkerMapper "+_stringID);
- }
-
- //always reset stats because counters per map task (for case of JVM reuse)
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- {
- CacheStatistics.reset();
- Statistics.reset();
- }
- }
-
- /**
- *
- */
- @Override
- public void close()
- throws IOException
- {
- //cleanup cache and local tmp dir
- RemoteParForUtils.cleanupWorkingDirectories();
-
- //change cache status for jvm_reuse (make empty allows us to
- //reuse in-memory objects if still present, re-load from HDFS
- //if evicted by garbage collector - without this approach, we
- //could not cleanup the local working dir, because this would
- //delete evicted matrices as well.
- if( ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER )
- {
- for( RemoteParWorkerMapper pw : _sCache.values() )
- {
- LocalVariableMap vars = pw._ec.getVariables();
- for( String varName : vars.keySet() )
- {
- Data dat = vars.get(varName);
- if( dat instanceof MatrixObject )
- ((MatrixObject)dat).setEmptyStatus();
- }
- }
- }
-
- //ensure caching is not disabled for CP in local mode
- CacheableData.enableCaching();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMerge.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMerge.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMerge.java
deleted file mode 100644
index c4ff3c5..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMerge.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-
-/**
- * Due to independence of all iterations, any result has the following properties:
- * (1) non local var, (2) matrix object, and (3) completely independent.
- * These properties allow us to realize result merging in parallel without any synchronization.
- *
- */
-public abstract class ResultMerge
-{
-
- protected static final Log LOG = LogFactory.getLog(ResultMerge.class.getName());
-
- protected static final String NAME_SUFFIX = "_rm";
-
- //inputs to result merge
- protected MatrixObject _output = null;
- protected MatrixObject[] _inputs = null;
- protected String _outputFName = null;
-
- protected ResultMerge( )
- {
-
- }
-
- public ResultMerge( MatrixObject out, MatrixObject[] in, String outputFilename )
- {
- _output = out;
- _inputs = in;
- _outputFName = outputFilename;
- }
-
- /**
- * Merge all given input matrices sequentially into the given output matrix.
- * The required space in-memory is the size of the output matrix plus the size
- * of one input matrix at a time.
- *
- * @return output (merged) matrix
- * @throws DMLRuntimeException
- */
- public abstract MatrixObject executeSerialMerge()
- throws DMLRuntimeException;
-
- /**
- * Merge all given input matrices in parallel into the given output matrix.
- * The required space in-memory is the size of the output matrix plus the size
- * of all input matrices.
- *
- * @param par degree of parallelism
- * @return output (merged) matrix
- * @throws DMLRuntimeException
- */
- public abstract MatrixObject executeParallelMerge( int par )
- throws DMLRuntimeException;
-
- /**
- *
- * @param out initially empty block
- * @param in
- * @param appendOnly
- * @throws DMLRuntimeException
- */
- protected void mergeWithoutComp( MatrixBlock out, MatrixBlock in, boolean appendOnly )
- throws DMLRuntimeException
- {
- //pass through to matrix block operations
- out.merge(in, appendOnly);
- }
-
- /**
- * NOTE: append only not applicable for wiht compare because output must be populated with
- * initial state of matrix - with append, this would result in duplicates.
- *
- * @param out
- * @param in
- * @throws DMLRuntimeException
- */
- protected void mergeWithComp( MatrixBlock out, MatrixBlock in, double[][] compare )
- throws DMLRuntimeException
- {
- //Notes for result correctness:
- // * Always iterate over entire block in order to compare all values
- // (using sparse iterator would miss values set to 0)
- // * Explicit NaN awareness because for cases were original matrix contains
- // NaNs, since NaN != NaN, otherwise we would potentially overwrite results
-
- if( in.isInSparseFormat() ) //sparse input format
- {
- int rows = in.getNumRows();
- int cols = in.getNumColumns();
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- {
- double value = in.getValueSparseUnsafe(i,j); //input value
- if( (value != compare[i][j] && !Double.isNaN(value) ) //for new values only (div)
- || Double.isNaN(value) != Double.isNaN(compare[i][j]) ) //NaN awareness
- {
- out.quickSetValue( i, j, value );
- }
- }
- }
- else //dense input format
- {
- //for a merge this case will seldom happen, as each input MatrixObject
- //has at most 1/numThreads of all values in it.
- int rows = in.getNumRows();
- int cols = in.getNumColumns();
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- {
- double value = in.getValueDenseUnsafe(i,j); //input value
- if( (value != compare[i][j] && !Double.isNaN(value) ) //for new values only (div)
- || Double.isNaN(value) != Double.isNaN(compare[i][j]) ) //NaN awareness
- {
- out.quickSetValue( i, j, value );
- }
- }
- }
- }
-
- protected long computeNonZeros( MatrixObject out, ArrayList<MatrixObject> in )
- {
- MatrixCharacteristics mc = out.getMatrixCharacteristics();
- long outNNZ = mc.getNonZeros();
- long ret = outNNZ;
- for( MatrixObject tmp : in )
- {
- MatrixCharacteristics tmpmc = tmp.getMatrixCharacteristics();
- long inNNZ = tmpmc.getNonZeros();
- ret += (inNNZ - outNNZ);
- }
-
- return ret;
- }
-
- /**
- *
- * @param in
- * @return
- */
- protected ArrayList<MatrixObject> convertToList(MatrixObject[] in)
- {
- ArrayList<MatrixObject> ret = new ArrayList<MatrixObject>();
- for( MatrixObject mo : in )
- ret.add( mo );
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
deleted file mode 100644
index 745ae08..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-
-/**
- *
- */
-public class ResultMergeLocalAutomatic extends ResultMerge
-{
-
- private ResultMerge _rm = null;
-
- public ResultMergeLocalAutomatic( MatrixObject out, MatrixObject[] in, String outputFilename )
- {
- super( out, in, outputFilename );
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- Timing time = new Timing(true);
-
- MatrixCharacteristics mc = _output.getMatrixCharacteristics();
- long rows = mc.getRows();
- long cols = mc.getCols();
-
- if( OptimizerRuleBased.isInMemoryResultMerge(rows, cols, OptimizerUtils.getLocalMemBudget()) )
- _rm = new ResultMergeLocalMemory( _output, _inputs, _outputFName );
- else
- _rm = new ResultMergeLocalFile( _output, _inputs, _outputFName );
-
- MatrixObject ret = _rm.executeSerialMerge();
-
- LOG.trace("Automatic result merge ("+_rm.getClass().getName()+") executed in "+time.stop()+"ms.");
-
- return ret;
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- MatrixCharacteristics mc = _output.getMatrixCharacteristics();
- long rows = mc.getRows();
- long cols = mc.getCols();
-
- if( OptimizerRuleBased.isInMemoryResultMerge(par * rows, cols, OptimizerUtils.getLocalMemBudget()) )
- _rm = new ResultMergeLocalMemory( _output, _inputs, _outputFName );
- else
- _rm = new ResultMergeLocalFile( _output, _inputs, _outputFName );
-
- return _rm.executeParallelMerge(par);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
deleted file mode 100644
index 8020719..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ /dev/null
@@ -1,1174 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.Cell;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.io.MatrixReader;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-
-/**
- *
- * TODO potential extension: parallel merge (create individual staging files concurrently)
- *
- * NOTE: file merge typically used due to memory constraints - parallel merge would increase the memory
- * consumption again.
- */
-public class ResultMergeLocalFile extends ResultMerge
-{
-
- //NOTE: if we allow simple copies, this might result in a scattered file and many MR tasks for subsequent jobs
- public static final boolean ALLOW_COPY_CELLFILES = false;
-
- //internal comparison matrix
- private IDSequence _seq = null;
-
- public ResultMergeLocalFile( MatrixObject out, MatrixObject[] in, String outputFilename )
- {
- super( out, in, outputFilename );
-
- _seq = new IDSequence();
- }
-
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- MatrixObject moNew = null; //always create new matrix object (required for nested parallelism)
-
- //Timing time = null;
- LOG.trace("ResultMerge (local, file): Execute serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")");
- // time = new Timing();
- // time.start();
-
-
- try
- {
-
-
- //collect all relevant inputs
- ArrayList<MatrixObject> inMO = new ArrayList<MatrixObject>();
- for( MatrixObject in : _inputs )
- {
- //check for empty inputs (no iterations executed)
- if( in !=null && in != _output )
- {
- //ensure that input file resides on disk
- in.exportData();
-
- //add to merge list
- inMO.add( in );
- }
- }
-
- if( !inMO.isEmpty() )
- {
- //ensure that outputfile (for comparison) resides on disk
- _output.exportData();
-
- //actual merge
- merge( _outputFName, _output, inMO );
-
- //create new output matrix (e.g., to prevent potential export<->read file access conflict
- moNew = createNewMatrixObject( _output, inMO );
- }
- else
- {
- moNew = _output; //return old matrix, to prevent copy
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //LOG.trace("ResultMerge (local, file): Executed serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+") in "+time.stop()+"ms");
-
- return moNew;
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- //graceful degradation to serial merge
- return executeSerialMerge();
- }
-
- /**
- *
- * @param output
- * @param inMO
- * @return
- * @throws DMLRuntimeException
- */
- private MatrixObject createNewMatrixObject(MatrixObject output, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- String varName = _output.getVarName();
- ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
-
- MatrixObject moNew = new MatrixObject( vt, _outputFName );
- moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
- moNew.setDataType( DataType.MATRIX );
-
- //create deep copy of metadata obj
- MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
- OutputInfo oiOld = metadata.getOutputInfo();
- InputInfo iiOld = metadata.getInputInfo();
- MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
- mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
- mc.setNonZeros( computeNonZeros(output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
- moNew.setMetaData( meta );
-
- return moNew;
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void merge( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- OutputInfo oi = ((MatrixFormatMetaData)outMo.getMetaData()).getOutputInfo();
- boolean withCompare = ( outMo.getNnz() != 0 ); //if nnz exist or unknown (-1)
-
- if( oi == OutputInfo.TextCellOutputInfo )
- {
- if(withCompare)
- mergeTextCellWithComp(fnameNew, outMo, inMO);
- else
- mergeTextCellWithoutComp( fnameNew, outMo, inMO );
- }
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- {
- if(withCompare)
- mergeBinaryCellWithComp(fnameNew, outMo, inMO);
- else
- mergeBinaryCellWithoutComp( fnameNew, outMo, inMO );
- }
- else if( oi == OutputInfo.BinaryBlockOutputInfo )
- {
- if(withCompare)
- mergeBinaryBlockWithComp( fnameNew, outMo, inMO );
- else
- mergeBinaryBlockWithoutComp( fnameNew, outMo, inMO );
- }
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void mergeTextCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- if( ALLOW_COPY_CELLFILES )
- {
- copyAllFiles(fnameNew, inMO);
- return; //we're done
- }
-
- //actual merge
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
-
- String valueStr = null;
-
- try
- {
- for( MatrixObject in : inMO ) //read/write all inputs
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+") via stream merge");
-
- JobConf tmpJob = new JobConf(ConfigurationManager.getCachedJobConf());
- Path tmpPath = new Path(in.getFileName());
- FileInputFormat.addInputPath(tmpJob, tmpPath);
- TextInputFormat informat = new TextInputFormat();
- informat.configure(tmpJob);
- InputSplit[] splits = informat.getSplits(tmpJob, 1);
-
- LongWritable key = new LongWritable();
- Text value = new Text();
-
- for(InputSplit split: splits)
- {
- RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, tmpJob, Reporter.NULL);
- try
- {
- while(reader.next(key, value))
- {
- valueStr = value.toString().trim();
- out.write( valueStr+"\n" );
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
- }
- }
- finally
- {
- if( out != null )
- out.close();
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge text cell results.", ex);
- }
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void mergeTextCellWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
- String fnameStagingCompare = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
-
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- //Step 0) write compare blocks to staging area (if necessary)
- LOG.trace("ResultMerge (local, file): Create merge compare matrix for output "+outMo.getVarName()+" (fname="+outMo.getFileName()+")");
- createTextCellStagingFile(fnameStagingCompare, outMo, 0);
-
- //Step 1) read and write blocks to staging area
- for( MatrixObject in : inMO )
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+")");
-
- long ID = _seq.getNextID();
- createTextCellStagingFile( fnameStaging, in, ID );
- }
-
- //Step 2) read blocks, consolidate, and write to HDFS
- createTextCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge text cell results.", ex);
- }
-
- LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
- LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void mergeBinaryCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- if( ALLOW_COPY_CELLFILES )
- {
- copyAllFiles(fnameNew, inMO);
- return; //we're done
- }
-
- //actual merge
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
- SequenceFile.Writer out = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class); //beware ca 50ms
-
- MatrixIndexes key = new MatrixIndexes();
- MatrixCell value = new MatrixCell();
-
- try
- {
- for( MatrixObject in : inMO ) //read/write all inputs
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+") via stream merge");
-
- JobConf tmpJob = new JobConf(ConfigurationManager.getCachedJobConf());
- Path tmpPath = new Path(in.getFileName());
-
- for(Path lpath : MatrixReader.getSequenceFilePaths(fs, tmpPath) )
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,tmpJob);
- try
- {
- while(reader.next(key, value))
- {
- out.append(key, value);
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
- }
- }
- finally
- {
- if( out != null )
- out.close();
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge binary cell results.", ex);
- }
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void mergeBinaryCellWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
- String fnameStagingCompare = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
-
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- //Step 0) write compare blocks to staging area (if necessary)
- LOG.trace("ResultMerge (local, file): Create merge compare matrix for output "+outMo.getVarName()+" (fname="+outMo.getFileName()+")");
- createBinaryCellStagingFile(fnameStagingCompare, outMo, 0);
-
- //Step 1) read and write blocks to staging area
- for( MatrixObject in : inMO )
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+")");
-
- long ID = _seq.getNextID();
- createBinaryCellStagingFile( fnameStaging, in, ID );
- }
-
- //Step 2) read blocks, consolidate, and write to HDFS
- createBinaryCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge binary cell results.", ex);
- }
-
- LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
- LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void mergeBinaryBlockWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
-
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- //Step 1) read and write blocks to staging area
- for( MatrixObject in : inMO )
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+")");
-
- createBinaryBlockStagingFile( fnameStaging, in );
- }
-
- //Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), false);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge binary block results.", ex);
- }
-
- LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
- }
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
- private void mergeBinaryBlockWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
- throws DMLRuntimeException
- {
- String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
- String fnameStagingCompare = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
-
- try
- {
- //delete target file if already exists
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
-
- //Step 0) write compare blocks to staging area (if necessary)
- LOG.trace("ResultMerge (local, file): Create merge compare matrix for output "+outMo.getVarName()+" (fname="+outMo.getFileName()+")");
-
- createBinaryBlockStagingFile(fnameStagingCompare, outMo);
-
- //Step 1) read and write blocks to staging area
- for( MatrixObject in : inMO )
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+")");
- createBinaryBlockStagingFile( fnameStaging, in );
- }
-
- //Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to merge binary block results.", ex);
- }
-
- LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
- LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
- }
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- private void createBinaryBlockStagingFile( String fnameStaging, MatrixObject mo )
- throws IOException
- {
- MatrixIndexes key = new MatrixIndexes();
- MatrixBlock value = new MatrixBlock();
-
- JobConf tmpJob = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(tmpJob);
- Path tmpPath = new Path(mo.getFileName());
-
- for(Path lpath : MatrixReader.getSequenceFilePaths(fs, tmpPath))
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,tmpJob);
- try
- {
- while(reader.next(key, value)) //for each block
- {
- String lname = key.getRowIndex()+"_"+key.getColumnIndex();
- String dir = fnameStaging+"/"+lname;
- if( value.getNonZeros()>0 ) //write only non-empty blocks
- {
- LocalFileUtils.checkAndCreateStagingDir( dir );
- LocalFileUtils.writeMatrixBlockToLocal(dir+"/"+_seq.getNextID(), value);
- }
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
- }
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @param ID
- * @throws IOException
- * @throws DMLRuntimeException
- */
-
- private void createTextCellStagingFile( String fnameStaging, MatrixObject mo, long ID )
- throws IOException, DMLRuntimeException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(mo.getFileName());
- FileInputFormat.addInputPath(job, path);
- TextInputFormat informat = new TextInputFormat();
- informat.configure(job);
- InputSplit[] splits = informat.getSplits(job, 1);
-
- LinkedList<Cell> buffer = new LinkedList<Cell>();
- LongWritable key = new LongWritable();
- Text value = new Text();
-
- MatrixCharacteristics mc = mo.getMatrixCharacteristics();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
- //long row = -1, col = -1; //FIXME needs reconsideration whenever textcell is used actively
- //NOTE MB: Originally, we used long row, col but this led reproducibly to JIT compilation
- // errors during runtime; experienced under WINDOWS, Intel x86-64, IBM JDK 64bit/32bit.
- // It works fine with int row, col but we require long for larger matrices.
- // Since, textcell is never used for result merge (hybrid/hadoop: binaryblock, singlenode:binarycell)
- // we just propose the to exclude it with -Xjit:exclude={package.method*}(count=0,optLevel=0)
-
- FastStringTokenizer st = new FastStringTokenizer(' ');
-
- for(InputSplit split : splits)
- {
- RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
- try
- {
- while(reader.next(key, value))
- {
- st.reset( value.toString() ); //reset tokenizer
- long row = st.nextLong();
- long col = st.nextLong();
- double lvalue = Double.parseDouble( st.nextToken() );
-
- Cell tmp = new Cell( row, col, lvalue );
-
- buffer.addLast( tmp );
- if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE ) //periodic flush
- {
- appendCellBufferToStagingArea(fnameStaging, ID, buffer, brlen, bclen);
- buffer.clear();
- }
- }
-
- //final flush
- if( !buffer.isEmpty() )
- {
- appendCellBufferToStagingArea(fnameStaging, ID, buffer, brlen, bclen);
- buffer.clear();
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
- }
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @param ID
- * @throws IOException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void createBinaryCellStagingFile( String fnameStaging, MatrixObject mo, long ID )
- throws IOException, DMLRuntimeException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- Path path = new Path(mo.getFileName());
- FileSystem fs = FileSystem.get(job);
-
- LinkedList<Cell> buffer = new LinkedList<Cell>();
- MatrixIndexes key = new MatrixIndexes();
- MatrixCell value = new MatrixCell();
-
- MatrixCharacteristics mc = mo.getMatrixCharacteristics();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
-
- for(Path lpath: MatrixReader.getSequenceFilePaths(fs, path))
- {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
- try
- {
- while(reader.next(key, value))
- {
- Cell tmp = new Cell( key.getRowIndex(), key.getColumnIndex(), value.getValue() );
-
- buffer.addLast( tmp );
- if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE ) //periodic flush
- {
- appendCellBufferToStagingArea(fnameStaging, ID, buffer, brlen, bclen);
- buffer.clear();
- }
- }
-
- //final flush
- if( !buffer.isEmpty() )
- {
- appendCellBufferToStagingArea(fnameStaging, ID, buffer, brlen, bclen);
- buffer.clear();
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
- }
- }
-
- /**
- * @param fnameStaging
- * @param ID
- * @param buffer
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- */
- private void appendCellBufferToStagingArea( String fnameStaging, long ID, LinkedList<Cell> buffer, int brlen, int bclen )
- throws DMLRuntimeException, IOException
- {
- HashMap<Long,HashMap<Long,LinkedList<Cell>>> sortedBuffer = new HashMap<Long, HashMap<Long,LinkedList<Cell>>>();
- long brow, bcol, row_offset, col_offset;
-
- for( Cell c : buffer )
- {
- brow = (c.getRow()-1)/brlen + 1;
- bcol = (c.getCol()-1)/bclen + 1;
- row_offset = (brow-1)*brlen + 1;
- col_offset = (bcol-1)*bclen + 1;
-
- c.setRow( c.getRow() - row_offset);
- c.setCol(c.getCol() - col_offset);
-
- if( !sortedBuffer.containsKey(brow) )
- sortedBuffer.put(brow, new HashMap<Long,LinkedList<Cell>>());
- if( !sortedBuffer.get(brow).containsKey(bcol) )
- sortedBuffer.get(brow).put(bcol, new LinkedList<Cell>());
- sortedBuffer.get(brow).get(bcol).addLast(c);
- }
-
- //write lists of cells to local files
- for( Entry<Long,HashMap<Long,LinkedList<Cell>>> e : sortedBuffer.entrySet() )
- {
- brow = e.getKey();
- for( Entry<Long,LinkedList<Cell>> e2 : e.getValue().entrySet() )
- {
- bcol = e2.getKey();
- String lname = brow+"_"+bcol;
- String dir = fnameStaging+"/"+lname;
- LocalFileUtils.checkAndCreateStagingDir( dir );
- StagingFileUtils.writeCellListToLocal(dir+"/"+ID, e2.getValue());
- }
- }
- }
-
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
- throws IOException, DMLRuntimeException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
-
- MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
- long rlen = mc.getRows();
- long clen = mc.getCols();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
-
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixBlock.class); //beware ca 50ms
- try
- {
- MatrixIndexes indexes = new MatrixIndexes();
- for(long brow = 1; brow <= (long)Math.ceil(rlen/(double)brlen); brow++)
- for(long bcol = 1; bcol <= (long)Math.ceil(clen/(double)bclen); bcol++)
- {
- File dir = new File(fnameStaging+"/"+brow+"_"+bcol);
- File dir2 = new File(fnameStagingCompare+"/"+brow+"_"+bcol);
- MatrixBlock mb = null;
-
- if( dir.exists() )
- {
- if( withCompare && dir2.exists() ) //WITH COMPARE BLOCK
- {
- //copy only values that are different from the original
- String[] lnames2 = dir2.list();
- if( lnames2.length != 1 ) //there should be exactly 1 compare block
- throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
- mb = LocalFileUtils.readMatrixBlockFromLocal( dir2+"/"+lnames2[0] );
- boolean appendOnly = mb.isInSparseFormat();
- double[][] compare = DataConverter.convertToDoubleMatrix(mb);
-
- String[] lnames = dir.list();
- for( String lname : lnames )
- {
- MatrixBlock tmp = LocalFileUtils.readMatrixBlockFromLocal( dir+"/"+lname );
- mergeWithComp(mb, tmp, compare);
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- else //WITHOUT COMPARE BLOCK
- {
- //copy all non-zeros from all workers
- String[] lnames = dir.list();
- boolean appendOnly = false;
- for( String lname : lnames )
- {
- if( mb == null )
- {
- mb = LocalFileUtils.readMatrixBlockFromLocal( dir+"/"+lname );
- appendOnly = mb.isInSparseFormat();
- }
- else
- {
- MatrixBlock tmp = LocalFileUtils.readMatrixBlockFromLocal( dir+"/"+lname );
- mergeWithoutComp(mb, tmp, appendOnly);
- }
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- }
- else
- {
- //NOTE: whenever runtime does not need all blocks anymore, this can be removed
- int maxRow = (int)(((brow-1)*brlen + brlen < rlen) ? brlen : rlen - (brow-1)*brlen);
- int maxCol = (int)(((bcol-1)*bclen + bclen < clen) ? bclen : clen - (bcol-1)*bclen);
-
- mb = new MatrixBlock(maxRow, maxCol, true);
- }
-
- //mb.examSparsity(); //done on write anyway and mb not reused
- indexes.setIndexes(brow, bcol);
- writer.append(indexes, mb);
- }
- }
- finally
- {
- if( writer != null )
- writer.close();
- }
- }
-
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
- private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
- throws IOException, DMLRuntimeException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
-
- MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
- long rlen = mc.getRows();
- long clen = mc.getCols();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
-
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
- try
- {
- //for obj reuse and preventing repeated buffer re-allocations
- StringBuilder sb = new StringBuilder();
-
- boolean written=false;
- for(long brow = 1; brow <= (long)Math.ceil(rlen/(double)brlen); brow++)
- for(long bcol = 1; bcol <= (long)Math.ceil(clen/(double)bclen); bcol++)
- {
- File dir = new File(fnameStaging+"/"+brow+"_"+bcol);
- File dir2 = new File(fnameStagingCompare+"/"+brow+"_"+bcol);
- MatrixBlock mb = null;
-
- long row_offset = (brow-1)*brlen + 1;
- long col_offset = (bcol-1)*bclen + 1;
-
-
- if( dir.exists() )
- {
- if( withCompare && dir2.exists() ) //WITH COMPARE BLOCK
- {
- //copy only values that are different from the original
- String[] lnames2 = dir2.list();
- if( lnames2.length != 1 ) //there should be exactly 1 compare block
- throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
- mb = StagingFileUtils.readCellList2BlockFromLocal( dir2+"/"+lnames2[0], brlen, bclen );
- boolean appendOnly = mb.isInSparseFormat();
- double[][] compare = DataConverter.convertToDoubleMatrix(mb);
-
- String[] lnames = dir.list();
- for( String lname : lnames )
- {
- MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- mergeWithComp(mb, tmp, compare);
- }
-
- //sort sparse and exam sparsity due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- else //WITHOUT COMPARE BLOCK
- {
- //copy all non-zeros from all workers
- String[] lnames = dir.list();
- boolean appendOnly = false;
- for( String lname : lnames )
- {
- if( mb == null )
- {
- mb = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- appendOnly = mb.isInSparseFormat();
- }
- else
- {
- MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- mergeWithoutComp(mb, tmp, appendOnly);
- }
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- }
-
- //write the block to text cell
- if( mb!=null )
- {
- if( mb.isInSparseFormat() )
- {
- SparseRowsIterator iter = mb.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- sb.append(row_offset+lcell.i);
- sb.append(' ');
- sb.append(col_offset+lcell.j);
- sb.append(' ');
- sb.append(lcell.v);
- sb.append('\n');
- out.write( sb.toString() );
- sb.setLength(0);
- written = true;
- }
- }
- else
- {
- for( int i=0; i<brlen; i++ )
- for( int j=0; j<bclen; j++ )
- {
- double lvalue = mb.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //for nnz
- {
- sb.append(row_offset+i);
- sb.append(' ');
- sb.append(col_offset+j);
- sb.append(' ');
- sb.append(lvalue);
- sb.append('\n');
- out.write( sb.toString() );
- sb.setLength(0);
- written = true;
- }
- }
- }
- }
- }
-
- if( !written )
- out.write("1 1 0\n");
- }
- finally
- {
- if( out != null )
- out.close();
- }
- }
-
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("deprecation")
- private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
- throws IOException, DMLRuntimeException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
-
- MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
- long rlen = mc.getRows();
- long clen = mc.getCols();
- int brlen = mc.getRowsPerBlock();
- int bclen = mc.getColsPerBlock();
-
-
- MatrixIndexes indexes = new MatrixIndexes(1,1);
- MatrixCell cell = new MatrixCell(0);
-
- SequenceFile.Writer out = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class); //beware ca 50ms
- try
- {
- boolean written=false;
- for(long brow = 1; brow <= (long)Math.ceil(rlen/(double)brlen); brow++)
- for(long bcol = 1; bcol <= (long)Math.ceil(clen/(double)bclen); bcol++)
- {
- File dir = new File(fnameStaging+"/"+brow+"_"+bcol);
- File dir2 = new File(fnameStagingCompare+"/"+brow+"_"+bcol);
- MatrixBlock mb = null;
-
- long row_offset = (brow-1)*brlen + 1;
- long col_offset = (bcol-1)*bclen + 1;
-
-
- if( dir.exists() )
- {
- if( withCompare && dir2.exists() ) //WITH COMPARE BLOCK
- {
- //copy only values that are different from the original
- String[] lnames2 = dir2.list();
- if( lnames2.length != 1 ) //there should be exactly 1 compare block
- throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
- mb = StagingFileUtils.readCellList2BlockFromLocal( dir2+"/"+lnames2[0], brlen, bclen );
- boolean appendOnly = mb.isInSparseFormat();
- double[][] compare = DataConverter.convertToDoubleMatrix(mb);
-
- String[] lnames = dir.list();
- for( String lname : lnames )
- {
- MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- mergeWithComp(mb, tmp, compare);
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- else //WITHOUT COMPARE BLOCK
- {
- //copy all non-zeros from all workers
- String[] lnames = dir.list();
- boolean appendOnly = false;
- for( String lname : lnames )
- {
- if( mb == null )
- {
- mb = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- appendOnly = mb.isInSparseFormat();
- }
- else
- {
- MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal( dir+"/"+lname, brlen, bclen );
- mergeWithoutComp(mb, tmp, appendOnly);
- }
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mb.sortSparseRows();
-
- //change sparsity if required after
- mb.examSparsity();
- }
- }
-
- //write the block to binary cell
- if( mb!=null )
- {
- if( mb.isInSparseFormat() )
- {
- SparseRowsIterator iter = mb.getSparseRowsIterator();
- while( iter.hasNext() )
- {
- IJV lcell = iter.next();
- indexes.setIndexes(row_offset+lcell.i, col_offset+lcell.j);
- cell.setValue(lcell.v);
- out.append(indexes,cell);
- written = true;
- }
- }
- else
- {
- for( int i=0; i<brlen; i++ )
- for( int j=0; j<bclen; j++ )
- {
- double lvalue = mb.getValueDenseUnsafe(i, j);
- if( lvalue != 0 ) //for nnz
- {
- indexes.setIndexes(row_offset+i, col_offset+j);
- cell.setValue(lvalue);
- out.append(indexes,cell);
- written = true;
- }
- }
- }
- }
- }
-
- if( !written )
- out.append(indexes,cell);
- }
- finally
- {
- if( out != null )
- out.close();
- }
- }
-
- /**
- *
- * @param fnameNew
- * @param inMO
- * @throws CacheException
- * @throws IOException
- */
- private void copyAllFiles( String fnameNew, ArrayList<MatrixObject> inMO )
- throws CacheException, IOException
- {
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- FileSystem fs = FileSystem.get(job);
- Path path = new Path( fnameNew );
-
- //create output dir
- fs.mkdirs(path);
-
- //merge in all input matrix objects
- IDSequence seq = new IDSequence();
- for( MatrixObject in : inMO )
- {
- LOG.trace("ResultMerge (local, file): Merge input "+in.getVarName()+" (fname="+in.getFileName()+") via file rename.");
-
- //copy over files (just rename file or entire dir)
- Path tmpPath = new Path(in.getFileName());
- String lname = tmpPath.getName();
- fs.rename(tmpPath, new Path(fnameNew+"/"+lname+seq.getNextID()));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
deleted file mode 100644
index 77b3bb5..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- * Local in-memory realization of result merge. If the resulting matrix is
- * small enough to fit into the JVM memory, this class can be used for efficient
- * serial or multi-threaded merge.
- *
- *
- */
-public class ResultMergeLocalMemory extends ResultMerge
-{
-
- //internal comparison matrix
- private double[][] _compare = null;
-
- public ResultMergeLocalMemory( MatrixObject out, MatrixObject[] in, String outputFilename )
- {
- super( out, in, outputFilename );
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- MatrixObject moNew = null; //always create new matrix object (required for nested parallelism)
-
- LOG.trace("ResultMerge (local, in-memory): Execute serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")");
-
- try
- {
- //get matrix blocks through caching
- MatrixBlock outMB = _output.acquireRead();
-
- //get old output matrix from cache for compare
- int estnnz = outMB.getNumRows()*outMB.getNumColumns();
- MatrixBlock outMBNew = new MatrixBlock(outMB.getNumRows(), outMB.getNumColumns(),
- outMB.isInSparseFormat(), estnnz);
- boolean appendOnly = outMBNew.isInSparseFormat();
-
- //create compare matrix if required (existing data in result)
- _compare = createCompareMatrix(outMB);
- if( _compare != null )
- outMBNew.copy(outMB);
-
- //serial merge all inputs
- boolean flagMerged = false;
- for( MatrixObject in : _inputs )
- {
- //check for empty inputs (no iterations executed)
- if( in !=null && in != _output )
- {
- LOG.trace("ResultMerge (local, in-memory): Merge input "+in.getVarName()+" (fname="+in.getFileName()+")");
-
- //read/pin input_i
- MatrixBlock inMB = in.acquireRead();
-
- //core merge
- merge( outMBNew, inMB, appendOnly );
-
- //unpin and clear in-memory input_i
- in.release();
- in.clearData();
- flagMerged = true;
-
- //determine need for sparse2dense change during merge
- boolean sparseToDense = appendOnly && !MatrixBlock.evalSparseFormatInMemory(
- outMBNew.getNumRows(), outMBNew.getNumColumns(), outMBNew.getNonZeros());
- if( sparseToDense ) {
- outMBNew.sortSparseRows(); //sort sparse due to append-only
- outMBNew.examSparsity(); //sparse-dense representation change
- appendOnly = false; //change merge state for subsequent inputs
- }
- }
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- outMBNew.sortSparseRows();
-
- //change sparsity if required after
- outMBNew.examSparsity();
-
- //create output
- if( flagMerged )
- {
- //create new output matrix
- //(e.g., to prevent potential export<->read file access conflict in specific cases of
- // local-remote nested parfor))
- moNew = createNewMatrixObject( outMBNew );
- }
- else
- {
- moNew = _output; //return old matrix, to prevent copy
- }
-
- //release old output, and all inputs
- _output.release();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //LOG.trace("ResultMerge (local, in-memory): Executed serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+") in "+time.stop()+"ms");
-
- return moNew;
- }
-
- @Override
- public MatrixObject executeParallelMerge( int par )
- throws DMLRuntimeException
- {
- MatrixObject moNew = null; //always create new matrix object (required for nested parallelism)
-
- //Timing time = null;
- LOG.trace("ResultMerge (local, in-memory): Execute parallel (par="+par+") merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")");
- // time = new Timing();
- // time.start();
-
-
- try
- {
- //get matrix blocks through caching
- MatrixBlock outMB = _output.acquireRead();
- ArrayList<MatrixObject> inMO = new ArrayList<MatrixObject>();
- for( MatrixObject in : _inputs )
- {
- //check for empty inputs (no iterations executed)
- if( in !=null && in != _output )
- inMO.add( in );
- }
-
- if( !inMO.isEmpty() ) //if there exist something to merge
- {
- //get old output matrix from cache for compare
- //NOTE: always in dense representation in order to allow for parallel unsynchronized access
- long rows = outMB.getNumRows();
- long cols = outMB.getNumColumns();
- MatrixBlock outMBNew = new MatrixBlock((int)rows, (int)cols, false);
- outMBNew.allocateDenseBlockUnsafe((int)rows, (int)cols);
-
- //create compare matrix if required (existing data in result)
- _compare = createCompareMatrix(outMB);
- if( _compare != null )
- outMBNew.copy(outMB);
-
- //parallel merge of all inputs
-
- int numThreads = Math.min(par, inMO.size()); //number of inputs can be lower than par
- numThreads = Math.min(numThreads, InfrastructureAnalyzer.getLocalParallelism()); //ensure robustness for remote exec
- Thread[] threads = new Thread[ numThreads ];
- for( int k=0; k<inMO.size(); k+=numThreads ) //multiple waves if necessary
- {
- //create and start threads
- for( int i=0; i<threads.length; i++ )
- {
- ResultMergeWorker rmw = new ResultMergeWorker(inMO.get(k+i), outMBNew);
- threads[i] = new Thread(rmw);
- threads[i].setPriority(Thread.MAX_PRIORITY);
- threads[i].start(); // start execution
- }
- //wait for all workers to finish
- for( int i=0; i<threads.length; i++ )
- {
- threads[i].join();
- }
- }
-
- //create new output matrix
- //(e.g., to prevent potential export<->read file access conflict in specific cases of
- // local-remote nested parfor))
- moNew = createNewMatrixObject( outMBNew );
- }
- else
- {
- moNew = _output; //return old matrix, to prevent copy
- }
-
- //release old output, and all inputs
- _output.release();
- //_output.clearData(); //save, since it respects pin/unpin
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //LOG.trace("ResultMerge (local, in-memory): Executed parallel (par="+par+") merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+") in "+time.stop()+"ms");
-
- return moNew;
- }
-
- /**
- *
- * @param output
- * @return
- */
- private double[][] createCompareMatrix( MatrixBlock output )
- {
- double[][] ret = null;
-
- //create compare matrix only if required
- if( output.getNonZeros() > 0 )
- {
- ret = DataConverter.convertToDoubleMatrix( output );
- }
-
- return ret;
- }
-
- /**
- *
- * @param varName
- * @param vt
- * @param metadata
- * @param data
- * @return
- * @throws DMLRuntimeException
- */
- private MatrixObject createNewMatrixObject( MatrixBlock data )
- throws DMLRuntimeException
- {
- String varName = _output.getVarName();
- ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
-
- MatrixObject moNew = new MatrixObject( vt, _outputFName );
- moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
- moNew.setDataType( DataType.MATRIX );
-
- //create deep copy of metadata obj
- MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
- OutputInfo oiOld = metadata.getOutputInfo();
- InputInfo iiOld = metadata.getInputInfo();
- MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
- mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
- mc.setNonZeros(data.getNonZeros());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
- moNew.setMetaData( meta );
-
- //adjust dense/sparse representation
- data.examSparsity();
-
- //release new output
- moNew.acquireModify(data);
- moNew.release();
-
- return moNew;
- }
-
-
- /**
- * Merges <code>in</code> into <code>out</code> by inserting all non-zeros of <code>in</code>
- * into <code>out</code> at their given positions. This is an update-in-place.
- *
- * NOTE: similar to converters, but not directly applicable as we are interested in combining
- * two objects with each other; not unary transformation.
- *
- * @param out
- * @param in
- * @throws DMLRuntimeException
- */
- private void merge( MatrixBlock out, MatrixBlock in, boolean appendOnly )
- throws DMLRuntimeException
- {
- if( _compare == null )
- mergeWithoutComp(out, in, appendOnly);
- else
- mergeWithComp(out, in, _compare);
- }
-
-
- /**
- * NOTE: only used if matrix in dense
- */
- private class ResultMergeWorker implements Runnable
- {
- private MatrixObject _inMO = null;
- private MatrixBlock _outMB = null;
-
- public ResultMergeWorker(MatrixObject inMO, MatrixBlock outMB)
- {
- _inMO = inMO;
- _outMB = outMB;
- }
-
- @Override
- public void run()
- {
- //read each input if required
- try
- {
- LOG.trace("ResultMerge (local, in-memory): Merge input "+_inMO.getVarName()+" (fname="+_inMO.getFileName()+")");
-
- MatrixBlock inMB = _inMO.acquireRead(); //incl. implicit read from HDFS
- merge( _outMB, inMB, false );
- _inMO.release();
- _inMO.clearData();
- }
- catch(Exception ex)
- {
- throw new RuntimeException("Failed to parallel merge result.", ex);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
deleted file mode 100644
index 041b0bd..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-
-public class ResultMergeRemoteGrouping extends WritableComparator
-{
-
-
- protected ResultMergeRemoteGrouping()
- {
- super(ResultMergeTaggedMatrixIndexes.class,true);
- }
-
- @SuppressWarnings("rawtypes")
- @Override
- public int compare(WritableComparable k1, WritableComparable k2)
- {
- ResultMergeTaggedMatrixIndexes key1 = (ResultMergeTaggedMatrixIndexes)k1;
- ResultMergeTaggedMatrixIndexes key2 = (ResultMergeTaggedMatrixIndexes)k2;
-
- //group by matrix indexes only (including all tags)
- return key1.getIndexes().compareTo(key2.getIndexes());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
deleted file mode 100644
index 0a406e3..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.StagingFileUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * MR job class for submitting parfor result merge MR jobs.
- *
- */
-public class ResultMergeRemoteMR extends ResultMerge
-{
-
- public static final byte COMPARE_TAG = 'c';
- public static final byte DATA_TAG = 'd';
-
- private long _pfid = -1;
- private int _numMappers = -1;
- private int _numReducers = -1;
- private int _replication = -1;
- //private int _max_retry = -1;
- private boolean _jvmReuse = false;
-
- public ResultMergeRemoteMR(MatrixObject out, MatrixObject[] in, String outputFilename, long pfid, int numMappers, int numReducers, int replication, int max_retry, boolean jvmReuse)
- {
- super(out, in, outputFilename);
-
- _pfid = pfid;
- _numMappers = numMappers;
- _numReducers = numReducers;
- _replication = replication;
- //_max_retry = max_retry;
- _jvmReuse = jvmReuse;
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- //graceful degradation to parallel merge
- return executeParallelMerge( _numMappers );
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- MatrixObject moNew = null; //always create new matrix object (required for nested parallelism)
-
- //Timing time = null;
- LOG.trace("ResultMerge (remote, mr): Execute serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")");
- // time = new Timing();
- // time.start();
-
-
- try
- {
- //collect all relevant inputs
- Collection<String> srcFnames = new LinkedList<String>();
- ArrayList<MatrixObject> inMO = new ArrayList<MatrixObject>();
- for( MatrixObject in : _inputs )
- {
- //check for empty inputs (no iterations executed)
- if( in !=null && in != _output )
- {
- //ensure that input file resides on disk
- in.exportData();
-
- //add to merge list
- srcFnames.add( in.getFileName() );
- inMO.add(in);
- }
- }
-
- if( !srcFnames.isEmpty() )
- {
- //ensure that outputfile (for comparison) resides on disk
- _output.exportData();
-
- //actual merge
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
- MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
-
- String fnameCompare = _output.getFileName();
- if( mcOld.getNonZeros()==0 )
- fnameCompare = null; //no compare required
-
- executeMerge(fnameCompare, _outputFName, srcFnames.toArray(new String[0]),
- metadata.getInputInfo(),metadata.getOutputInfo(), mcOld.getRows(), mcOld.getCols(),
- mcOld.getRowsPerBlock(), mcOld.getColsPerBlock());
-
- //create new output matrix (e.g., to prevent potential export<->read file access conflict
- String varName = _output.getVarName();
- ValueType vt = _output.getValueType();
- moNew = new MatrixObject( vt, _outputFName );
- moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
- moNew.setDataType( DataType.MATRIX );
- OutputInfo oiOld = metadata.getOutputInfo();
- InputInfo iiOld = metadata.getInputInfo();
- MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
- mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
- mc.setNonZeros( computeNonZeros(_output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
- moNew.setMetaData( meta );
- }
- else
- {
- moNew = _output; //return old matrix, to prevent copy
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- //LOG.trace("ResultMerge (local, file): Executed serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+") in "+time.stop()+"ms");
-
- return moNew;
- }
-
- /**
- *
- * @param fname null if no comparison required
- * @param fnameNew
- * @param srcFnames
- * @param ii
- * @param oi
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- @SuppressWarnings({ "unused", "deprecation" })
- protected void executeMerge(String fname, String fnameNew, String[] srcFnames, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen)
- throws DMLRuntimeException
- {
- String jobname = "ParFor-RMMR";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- JobConf job;
- job = new JobConf( ResultMergeRemoteMR.class );
- job.setJobName(jobname+_pfid);
-
- //maintain dml script counters
- Statistics.incrementNoOfCompiledMRJobs();
-
- //warning for textcell/binarycell without compare
- boolean withCompare = (fname!=null);
- if( (oi == OutputInfo.TextCellOutputInfo || oi == OutputInfo.BinaryCellOutputInfo) && !withCompare && ResultMergeLocalFile.ALLOW_COPY_CELLFILES )
- LOG.warn("Result merge for "+OutputInfo.outputInfoToString(oi)+" without compare can be realized more efficiently with LOCAL_FILE than REMOTE_MR.");
-
- try
- {
- Path pathCompare = null;
- Path pathNew = new Path(fnameNew);
-
- /////
- //configure the MR job
- if( withCompare ) {
- pathCompare = new Path(fname).makeQualified(FileSystem.get(job));
- MRJobConfiguration.setResultMergeInfo(job, pathCompare.toString(), ii, LocalFileUtils.getWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE), rlen, clen, brlen, bclen);
- }
- else
- MRJobConfiguration.setResultMergeInfo(job, "null", ii, LocalFileUtils.getWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE), rlen, clen, bclen, bclen);
-
-
- //set mappers, reducers, combiners
- job.setMapperClass(ResultMergeRemoteMapper.class);
- job.setReducerClass(ResultMergeRemoteReducer.class);
-
- if( oi == OutputInfo.TextCellOutputInfo )
- {
- job.setMapOutputKeyClass(MatrixIndexes.class);
- job.setMapOutputValueClass(TaggedMatrixCell.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Text.class);
- }
- else if( oi == OutputInfo.BinaryCellOutputInfo )
- {
- job.setMapOutputKeyClass(MatrixIndexes.class);
- job.setMapOutputValueClass(TaggedMatrixCell.class);
- job.setOutputKeyClass(MatrixIndexes.class);
- job.setOutputValueClass(MatrixCell.class);
- }
- else if ( oi == OutputInfo.BinaryBlockOutputInfo )
- {
- //setup partitioning, grouping, sorting for composite key (old API)
- job.setPartitionerClass(ResultMergeRemotePartitioning.class); //partitioning
- job.setOutputValueGroupingComparator(ResultMergeRemoteGrouping.class); //grouping
- job.setOutputKeyComparatorClass(ResultMergeRemoteSorting.class); //sorting
-
- job.setMapOutputKeyClass(ResultMergeTaggedMatrixIndexes.class);
- job.setMapOutputValueClass(TaggedMatrixBlock.class);
- job.setOutputKeyClass(MatrixIndexes.class);
- job.setOutputValueClass(MatrixBlock.class);
- }
-
- //set input format
- job.setInputFormat(ii.inputFormatClass);
-
- //set the input path
- Path[] paths = null;
- if( withCompare ) {
- paths= new Path[ srcFnames.length+1 ];
- paths[0] = pathCompare;
- for(int i=1; i<paths.length; i++)
- paths[i] = new Path( srcFnames[i-1] );
- }
- else {
- paths= new Path[ srcFnames.length ];
- for(int i=0; i<paths.length; i++)
- paths[i] = new Path( srcFnames[i] );
- }
- FileInputFormat.setInputPaths(job, paths);
-
- //set output format
- job.setOutputFormat(oi.outputFormatClass);
-
- //set output path
- MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
- FileOutputFormat.setOutputPath(job, pathNew);
-
-
- //////
- //set optimization parameters
-
- //set the number of mappers and reducers
- //job.setNumMapTasks( _numMappers ); //use default num mappers
- long reducerGroups = _numReducers;
- if( oi == OutputInfo.BinaryBlockOutputInfo )
- reducerGroups = Math.max(rlen/brlen,1) * Math.max(clen/bclen, 1);
- else //textcell/binarycell
- reducerGroups = Math.max((rlen*clen)/StagingFileUtils.CELL_BUFFER_SIZE, 1);
- job.setNumReduceTasks( (int)Math.min( _numReducers, reducerGroups) );
-
- //use FLEX scheduler configuration properties
- if( ParForProgramBlock.USE_FLEX_SCHEDULER_CONF )
- {
- job.setInt("flex.map.min", 0);
- job.setInt("flex.map.max", _numMappers);
- job.setInt("flex.reduce.min", 0);
- job.setInt("flex.reduce.max", _numMappers);
- }
-
- //disable automatic tasks timeouts and speculative task exec
- job.setInt("mapred.task.timeout", 0);
- job.setMapSpeculativeExecution(false);
-
- //set up preferred custom serialization framework for binary block format
- if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
- MRJobConfiguration.addBinaryBlockSerializationFramework( job );
-
- //enables the reuse of JVMs (multiple tasks per MR task)
- if( _jvmReuse )
- job.setNumTasksToExecutePerJvm(-1); //unlimited
-
- //enables compression - not conclusive for different codecs (empirically good compression ratio, but significantly slower)
- //job.set("mapred.compress.map.output", "true");
- //job.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
-
- //set the replication factor for the results
- job.setInt("dfs.replication", _replication);
-
- //set the max number of retries per map task
- // disabled job-level configuration to respect cluster configuration
- // note: this refers to hadoop2, hence it never had effect on mr1
- //job.setInt("mapreduce.map.maxattempts", _max_retry);
-
- //set unique working dir
- MRJobConfiguration.setUniqueWorkingDir(job);
-
- /////
- // execute the MR job
-
- JobClient.runJob(job);
-
- //maintain dml script counters
- Statistics.incrementNoOfExecutedMRJobs();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.maintainCPHeavyHitters("MR-Job_"+jobname, t1-t0);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
deleted file mode 100644
index 52690ee..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteMapper.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.FastStringTokenizer;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-
-/**
- * Remote resultmerge mapper implementation that does the preprocessing
- * in terms of tagging .
- *
- */
-public class ResultMergeRemoteMapper
- implements Mapper<Writable, Writable, Writable, Writable>
-{
-
- private ResultMergeMapper _mapper;
-
- public void map(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- //tag and pass-through matrix values
- _mapper.processKeyValue(key, value, out, reporter);
- }
-
- public void configure(JobConf job)
- {
- InputInfo ii = MRJobConfiguration.getResultMergeInputInfo(job);
- long[] tmp = MRJobConfiguration.getResultMergeMatrixCharacteristics( job );
- String compareFname = MRJobConfiguration.getResultMergeInfoCompareFilename(job);
- String currentFname = job.get("map.input.file");
-
- byte tag = 0;
- //startsWith comparison in order to account for part names in currentFname
- if( currentFname.startsWith(compareFname) )
- tag = ResultMergeRemoteMR.COMPARE_TAG;
- else
- tag = ResultMergeRemoteMR.DATA_TAG;
-
- if( ii == InputInfo.TextCellInputInfo )
- _mapper = new ResultMergeMapperTextCell(tag);
- else if( ii == InputInfo.BinaryCellInputInfo )
- _mapper = new ResultMergeMapperBinaryCell(tag);
- else if( ii == InputInfo.BinaryBlockInputInfo )
- _mapper = new ResultMergeMapperBinaryBlock(tag, tmp[0], tmp[1], tmp[2], tmp[3]);
- else
- throw new RuntimeException("Unable to configure mapper with unknown input info: "+ii.toString());
- }
-
- /**
- *
- */
- @Override
- public void close() throws IOException
- {
- //do nothing
- }
-
- private static abstract class ResultMergeMapper
- {
- protected byte _tag = 0;
-
- protected ResultMergeMapper( byte tag )
- {
- _tag = tag;
- }
-
- protected abstract void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException;
- }
-
- protected static class ResultMergeMapperTextCell extends ResultMergeMapper
- {
- private MatrixIndexes _objKey;
- private MatrixCell _objValueHelp;
- private TaggedMatrixCell _objValue;
- private FastStringTokenizer _st;
-
- protected ResultMergeMapperTextCell(byte tag)
- {
- super(tag);
- _objKey = new MatrixIndexes();
- _objValueHelp = new MatrixCell();
- _objValue = new TaggedMatrixCell();
- _objValue.setTag( _tag );
-
- _st = new FastStringTokenizer(' ');
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _st.reset( value.toString() ); //reset tokenizer
- long row = _st.nextLong();
- long col = _st.nextLong();
- double lvalue = _st.nextDouble();
-
- _objKey.setIndexes(row,col);
- _objValueHelp.setValue(lvalue);
- _objValue.setBaseObject(_objValueHelp);
-
- out.collect(_objKey, _objValue);
- }
- }
-
- protected static class ResultMergeMapperBinaryCell extends ResultMergeMapper
- {
- private TaggedMatrixCell _objValue;
-
- protected ResultMergeMapperBinaryCell(byte tag)
- {
- super(tag);
- _objValue = new TaggedMatrixCell();
- _objValue.setTag( _tag );
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _objValue.setBaseObject((MatrixCell)value);
- out.collect(key, _objValue);
- }
- }
-
- protected static class ResultMergeMapperBinaryBlock extends ResultMergeMapper
- {
- private ResultMergeTaggedMatrixIndexes _objKey;
- private TaggedMatrixBlock _objValue;
- private long _rlen = -1;
- private long _clen = -1;
- private long _brlen = -1;
- private long _bclen = -1;
-
- protected ResultMergeMapperBinaryBlock(byte tag, long rlen, long clen, long brlen, long bclen)
- {
- super(tag);
- _objKey = new ResultMergeTaggedMatrixIndexes();
- _objValue = new TaggedMatrixBlock();
- _objKey.setTag( _tag );
- _objValue.setTag( _tag );
-
- _rlen = rlen;
- _clen = clen;
- _brlen = brlen;
- _bclen = bclen;
- }
-
- @Override
- protected void processKeyValue(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- MatrixIndexes inkey = (MatrixIndexes)key;
- MatrixBlock inval = (MatrixBlock)value;
-
- //check valid block sizes
- if( inval.getNumRows() != UtilFunctions.computeBlockSize(_rlen, inkey.getRowIndex(), _brlen) )
- throw new IOException("Invalid number of rows for block "+inkey+": "+inval.getNumRows());
- if( inval.getNumColumns() != UtilFunctions.computeBlockSize(_clen, inkey.getColumnIndex(), _bclen) )
- throw new IOException("Invalid number of columns for block "+inkey+": "+inval.getNumColumns());
-
- //pass-through matrix blocks
- _objKey.getIndexes().setIndexes( inkey );
- _objValue.setBaseObject( inval );
- out.collect(_objKey, _objValue);
- }
- }
-}
[02/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
deleted file mode 100644
index d93949f..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemotePartitioning.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Partitioner;
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-
-public class ResultMergeRemotePartitioning implements Partitioner<ResultMergeTaggedMatrixIndexes, TaggedMatrixBlock>
-{
-
-
- private long _numColBlocks = -1;
-
-
- @Override
- public int getPartition(ResultMergeTaggedMatrixIndexes key, TaggedMatrixBlock val, int numPartitions)
- {
- //MB: Result merge might deal with lots of data but only few
- //different indexes (many worker result blocks for one final
- //result block). Hence, balanced partitioning it even more important
- //and unfortunately, our default hash function results in significant
- //load imbalance for those cases. Based on the known result dimensions
- //we can create a better partitioning scheme. However, it still makes
- //the assumption that there is no sparsity skew between blocks.
-
- MatrixIndexes ix = key.getIndexes();
- int blockid = (int) (ix.getRowIndex() * _numColBlocks + ix.getColumnIndex());
- int partition = blockid % numPartitions;
-
- //int hash = key.getIndexes().hashCode();
- //int partition = hash % numPartitions;
-
- return partition;
- }
-
- @Override
- public void configure(JobConf job)
- {
- long[] tmp = MRJobConfiguration.getResultMergeMatrixCharacteristics( job );
- long clen = tmp[1];
- int bclen = (int) tmp[3];
- _numColBlocks = clen/bclen + ((clen%bclen!=0)? 1 : 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
deleted file mode 100644
index 5066a82..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.TaggedMatrixCell;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- * Remote result merge reducer that receives all worker results partitioned by
- * cell index or blockindex and merges all results. Due to missing resettable iterators
- * in the old mapred API we need to spill parts of the value list to disk before merging
- * in case of binaryblock.
- *
- */
-public class ResultMergeRemoteReducer
- implements Reducer<Writable, Writable, Writable, Writable>
-{
-
- private ResultMergeReducer _reducer = null;
-
- public ResultMergeRemoteReducer( )
- {
-
- }
-
- @Override
- public void reduce(Writable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- _reducer.processKeyValueList(key, valueList, out, reporter);
- }
-
- /**
- *
- */
- public void configure(JobConf job)
- {
- InputInfo ii = MRJobConfiguration.getResultMergeInputInfo(job);
- String compareFname = MRJobConfiguration.getResultMergeInfoCompareFilename(job);
-
- //determine compare required
- boolean requiresCompare = false;
- if( !compareFname.equals("null") )
- requiresCompare = true;
-
- if( ii == InputInfo.TextCellInputInfo )
- _reducer = new ResultMergeReducerTextCell(requiresCompare);
- else if( ii == InputInfo.BinaryCellInputInfo )
- _reducer = new ResultMergeReducerBinaryCell(requiresCompare);
- else if( ii == InputInfo.BinaryBlockInputInfo )
- _reducer = new ResultMergeReducerBinaryBlock(requiresCompare);
- else
- throw new RuntimeException("Unable to configure mapper with unknown input info: "+ii.toString());
- }
-
- /**
- *
- */
- @Override
- public void close() throws IOException
- {
- //do nothing
- }
-
-
- private interface ResultMergeReducer //interface in order to allow ResultMergeReducerBinaryBlock to inherit from ResultMerge
- {
- void processKeyValueList( Writable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter )
- throws IOException;
- }
-
- private static class ResultMergeReducerTextCell implements ResultMergeReducer
- {
- private boolean _requiresCompare;
- private StringBuilder _sb = null;
- private Text _objValue = null;
-
- public ResultMergeReducerTextCell(boolean requiresCompare)
- {
- _requiresCompare = requiresCompare;
- _sb = new StringBuilder();
- _objValue = new Text();
- }
-
- @Override
- public void processKeyValueList(Writable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- //with compare
- if( _requiresCompare )
- {
- // NOTES MB:
- // 1) the old mapred api does not support multiple scans (reset/mark),
- // once we switch to the new api, we can use the resetableiterator for doing
- // the two required scans (finding the compare obj, compare all other objs)
- // 2) for 'textcell' we assume that the entire valueList fits into main memory
- // this is valid as we group by cells, i.e., we would need millions of input files
- // to exceed the usual 100-600MB per reduce task.
-
- //scan for compare object (incl result merge if compare available)
- MatrixIndexes key2 = (MatrixIndexes) key;
- Double cellCompare = null;
- Collection<Double> cellList = new LinkedList<Double>();
- boolean found = false;
- while( valueList.hasNext() ) {
- TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next();
- double lvalue = ((MatrixCell)tVal.getBaseObject()).getValue();
- if( tVal.getTag()==ResultMergeRemoteMR.COMPARE_TAG )
- cellCompare = lvalue;
- else
- {
- if( cellCompare == null )
- cellList.add( lvalue );
- else if( cellCompare.doubleValue()!=lvalue ) //compare on the fly
- {
- _sb.append(key2.getRowIndex());
- _sb.append(' ');
- _sb.append(key2.getColumnIndex());
- _sb.append(' ');
- _sb.append(lvalue);
- _objValue.set( _sb.toString() );
- _sb.setLength(0);
- out.collect(NullWritable.get(), _objValue );
- found = true;
- break; //only one write per cell possible (independence)
- }// note: objs with equal value are directly discarded
- }
- }
-
- //result merge for objs before compare
- if( !found )
- for( Double c : cellList )
- if( !c.equals( cellCompare ) )
- {
- _sb.append(key2.getRowIndex());
- _sb.append(' ');
- _sb.append(key2.getColumnIndex());
- _sb.append(' ');
- _sb.append(c.doubleValue());
- _objValue.set( _sb.toString() );
- _sb.setLength(0);
- out.collect(NullWritable.get(), _objValue );
- break; //only one write per cell possible (independence)
- }
- }
- //without compare
- else
- {
- MatrixIndexes key2 = (MatrixIndexes) key;
- while( valueList.hasNext() )
- {
- TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next();
- MatrixCell value = (MatrixCell) tVal.getBaseObject();
-
- _sb.append(key2.getRowIndex());
- _sb.append(' ');
- _sb.append(key2.getColumnIndex());
- _sb.append(' ');
- _sb.append(value.getValue());
- _objValue.set( _sb.toString() );
- _sb.setLength(0);
- out.collect(NullWritable.get(), _objValue );
- break; //only one write per cell possible (independence)
- }
- }
-
-
- }
- }
-
- private static class ResultMergeReducerBinaryCell implements ResultMergeReducer
- {
- private boolean _requiresCompare;
- private MatrixCell _objValue;
-
- public ResultMergeReducerBinaryCell(boolean requiresCompare)
- {
- _requiresCompare = requiresCompare;
- _objValue = new MatrixCell();
- }
-
- @Override
- public void processKeyValueList(Writable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- //with compare
- if( _requiresCompare )
- {
- // NOTES MB:
- // 1) the old mapred api does not support multiple scans (reset/mark),
- // once we switch to the new api, we can use the resetableiterator for doing
- // the two required scans (finding the compare obj, compare all other objs)
- // 2) for 'binarycell' we assume that the entire valueList fits into main memory
- // this is valid as we group by cells, i.e., we would need millions of input files
- // to exceed the usual 100-600MB per reduce task.
-
- //scan for compare object (incl result merge if compare available)
- Double cellCompare = null;
- Collection<Double> cellList = new LinkedList<Double>();
- boolean found = false;
- while( valueList.hasNext() ) {
- TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next();
- MatrixCell cVal = (MatrixCell) tVal.getBaseObject();
- if( tVal.getTag()==ResultMergeRemoteMR.COMPARE_TAG )
- cellCompare = cVal.getValue();
- else
- {
- if( cellCompare == null )
- cellList.add( cVal.getValue() );
- else if( cellCompare.doubleValue() != cVal.getValue() ) //compare on the fly
- {
- out.collect(key, cVal );
- found = true;
- break; //only one write per cell possible (independence)
- }// note: objs with equal value are directly discarded
- }
- }
-
- //result merge for objs before compare
- if( !found )
- for( Double c : cellList )
- if( !c.equals( cellCompare) )
- {
- _objValue.setValue(c.doubleValue());
- out.collect(key, _objValue );
- break; //only one write per cell possible (independence)
- }
- }
- //without compare
- else
- {
- while( valueList.hasNext() )
- {
- TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next();
- out.collect((MatrixIndexes)key, (MatrixCell)tVal.getBaseObject());
- break; //only one write per cell possible (independence)
- }
- }
- }
- }
-
- private static class ResultMergeReducerBinaryBlock extends ResultMerge implements ResultMergeReducer
- {
- private boolean _requiresCompare;
-
- public ResultMergeReducerBinaryBlock(boolean requiresCompare)
- {
- _requiresCompare = requiresCompare;
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- throw new DMLRuntimeException("Unsupported operation.");
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- throw new DMLRuntimeException("Unsupported operation.");
- }
-
- @Override
- public void processKeyValueList(Writable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter)
- throws IOException
- {
- try
- {
- MatrixIndexes ixOut = ((ResultMergeTaggedMatrixIndexes)key).getIndexes();
- MatrixBlock mbOut = null;
- double[][] aCompare = null;
- boolean appendOnly = false;
-
- //get and prepare compare block if required
- if( _requiresCompare )
- {
- TaggedMatrixBlock tVal = (TaggedMatrixBlock) valueList.next();
- MatrixBlock bVal = (MatrixBlock) tVal.getBaseObject();
- if( tVal.getTag()!=ResultMergeRemoteMR.COMPARE_TAG )
- throw new IOException("Failed to read compare block at expected first position.");
- aCompare = DataConverter.convertToDoubleMatrix(bVal);
- }
-
- //merge all result blocks into final result block
- while( valueList.hasNext() )
- {
- TaggedMatrixBlock tVal = (TaggedMatrixBlock) valueList.next();
- MatrixBlock bVal = (MatrixBlock) tVal.getBaseObject();
-
- if( mbOut == null ) //copy first block
- {
- mbOut = new MatrixBlock();
- mbOut.copy( bVal );
- appendOnly = mbOut.isInSparseFormat();
- }
- else //merge remaining blocks
- {
- if( _requiresCompare )
- mergeWithComp(mbOut, bVal, aCompare);
- else
- mergeWithoutComp(mbOut, bVal, appendOnly);
- }
- }
-
- //sort sparse due to append-only
- if( appendOnly )
- mbOut.sortSparseRows();
-
- //change sparsity if required after
- mbOut.examSparsity();
-
- out.collect(ixOut, mbOut);
- }
- catch( Exception ex )
- {
- throw new IOException(ex);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
deleted file mode 100644
index 4f81764..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-
-public class ResultMergeRemoteSorting extends WritableComparator
-{
-
-
- protected ResultMergeRemoteSorting()
- {
- super(ResultMergeTaggedMatrixIndexes.class, true);
- }
-
- @SuppressWarnings("rawtypes")
- @Override
- public int compare(WritableComparable k1, WritableComparable k2)
- {
- ResultMergeTaggedMatrixIndexes key1 = (ResultMergeTaggedMatrixIndexes)k1;
- ResultMergeTaggedMatrixIndexes key2 = (ResultMergeTaggedMatrixIndexes)k2;
-
- int ret = key1.getIndexes().compareTo(key2.getIndexes());
- if( ret == 0 ) //same indexes, secondary sort
- {
- ret = ((key1.getTag() == key2.getTag()) ? 0 :
- (key1.getTag() < key2.getTag())? -1 : 1);
- }
-
- return ret;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
deleted file mode 100644
index 6765bc3..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-
-import org.apache.spark.api.java.JavaPairRDD;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
-import com.ibm.bi.dml.runtime.instructions.spark.utils.RDDAggregateUtils;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- * MR job class for submitting parfor result merge MR jobs.
- *
- */
-public class ResultMergeRemoteSpark extends ResultMerge
-{
-
- private ExecutionContext _ec = null;
- private int _numMappers = -1;
- private int _numReducers = -1;
-
- public ResultMergeRemoteSpark(MatrixObject out, MatrixObject[] in, String outputFilename, ExecutionContext ec, int numMappers, int numReducers)
- {
- super(out, in, outputFilename);
-
- _ec = ec;
- _numMappers = numMappers;
- _numReducers = numReducers;
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- //graceful degradation to parallel merge
- return executeParallelMerge( _numMappers );
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- MatrixObject moNew = null; //always create new matrix object (required for nested parallelism)
-
- LOG.trace("ResultMerge (remote, spark): Execute serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")");
-
- try
- {
- if( _inputs != null && _inputs.length>0 )
- {
- //prepare compare
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
- MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
- MatrixObject compare = (mcOld.getNonZeros()==0) ? null : _output;
-
- //actual merge
- RDDObject ro = executeMerge(compare, _inputs, _output.getVarName(), mcOld.getRows(), mcOld.getCols(), mcOld.getRowsPerBlock(), mcOld.getColsPerBlock());
-
- //create new output matrix (e.g., to prevent potential export<->read file access conflict
- String varName = _output.getVarName();
- ValueType vt = _output.getValueType();
- moNew = new MatrixObject( vt, _outputFName );
- moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
- moNew.setDataType( DataType.MATRIX );
- OutputInfo oiOld = metadata.getOutputInfo();
- InputInfo iiOld = metadata.getInputInfo();
- MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
- mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
- mc.setNonZeros( computeNonZeros(_output, convertToList(_inputs)) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
- moNew.setMetaData( meta );
- moNew.setRDDHandle( ro );
- }
- else
- {
- moNew = _output; //return old matrix, to prevent copy
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- return moNew;
- }
-
- /**
- *
- * @param fname null if no comparison required
- * @param fnameNew
- * @param srcFnames
- * @param ii
- * @param oi
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("unchecked")
- protected RDDObject executeMerge(MatrixObject compare, MatrixObject[] inputs, String varname, long rlen, long clen, int brlen, int bclen)
- throws DMLRuntimeException
- {
- String jobname = "ParFor-RMSP";
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- SparkExecutionContext sec = (SparkExecutionContext)_ec;
- boolean withCompare = (compare!=null);
-
- RDDObject ret = null;
-
- //determine degree of parallelism
- int numRed = (int)determineNumReducers(rlen, clen, brlen, bclen, _numReducers);
-
- //sanity check for empty src files
- if( inputs == null || inputs.length==0 )
- throw new DMLRuntimeException("Execute merge should never be called with no inputs.");
-
- try
- {
- //Step 1: union over all results
- JavaPairRDD<MatrixIndexes, MatrixBlock> rdd = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- sec.getRDDHandleForMatrixObject(_inputs[0], InputInfo.BinaryBlockInputInfo);
- for( int i=1; i<_inputs.length; i++ ) {
- JavaPairRDD<MatrixIndexes, MatrixBlock> rdd2 = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- sec.getRDDHandleForMatrixObject(_inputs[i], InputInfo.BinaryBlockInputInfo);
- rdd = rdd.union(rdd2);
- }
-
- //Step 2a: merge with compare
- JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
- if( withCompare )
- {
- JavaPairRDD<MatrixIndexes, MatrixBlock> compareRdd = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
- sec.getRDDHandleForMatrixObject(compare, InputInfo.BinaryBlockInputInfo);
-
- //merge values which differ from compare values
- ResultMergeRemoteSparkWCompare cfun = new ResultMergeRemoteSparkWCompare();
- out = rdd.groupByKey(numRed) //group all result blocks per key
- .join(compareRdd) //join compare block and result blocks
- .mapToPair(cfun); //merge result blocks w/ compare
- }
- //Step 2b: merge without compare
- else
- {
- //direct merge in any order (disjointness guaranteed)
- out = RDDAggregateUtils.mergeByKey(rdd);
- }
-
- //Step 3: create output rdd handle w/ lineage
- ret = new RDDObject(out, varname);
- for( int i=0; i<_inputs.length; i++ ) {
- //child rdd handles guaranteed to exist
- RDDObject child = _inputs[i].getRDDHandle();
- ret.addLineageChild(child);
- }
- }
- catch( Exception ex )
- {
- throw new DMLRuntimeException(ex);
- }
-
- //maintain statistics
- Statistics.incrementNoOfCompiledSPInst();
- Statistics.incrementNoOfExecutedSPInst();
- if( DMLScript.STATISTICS ){
- Statistics.maintainCPHeavyHitters(jobname, System.nanoTime()-t0);
- }
-
- return ret;
- }
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param numRed
- * @return
- */
- private int determineNumReducers(long rlen, long clen, int brlen, int bclen, long numRed)
- {
- //set the number of mappers and reducers
- long reducerGroups = Math.max(rlen/brlen,1) * Math.max(clen/bclen, 1);
- int ret = (int)Math.min( numRed, reducerGroups );
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
deleted file mode 100644
index 7df2cd1..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.Iterator;
-
-import org.apache.spark.api.java.function.PairFunction;
-
-import scala.Tuple2;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.util.DataConverter;
-
-/**
- *
- */
-public class ResultMergeRemoteSparkWCompare extends ResultMerge implements PairFunction<Tuple2<MatrixIndexes,Tuple2<Iterable<MatrixBlock>,MatrixBlock>>, MatrixIndexes, MatrixBlock>
-{
-
- private static final long serialVersionUID = -5970805069405942836L;
-
- @Override
- public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, Tuple2<Iterable<MatrixBlock>, MatrixBlock>> arg)
- throws Exception
- {
- MatrixIndexes ixin = arg._1();
- Iterator<MatrixBlock> din = arg._2()._1().iterator();
- MatrixBlock cin = arg._2()._2();
-
- //create compare array
- double[][] compare = DataConverter.convertToDoubleMatrix(cin);
-
- //merge all blocks into compare block
- MatrixBlock out = new MatrixBlock(cin);
- while( din.hasNext() )
- mergeWithComp(out, din.next(), compare);
-
- //create output tuple
- return new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(ixin), out);
- }
-
- @Override
- public MatrixObject executeSerialMerge()
- throws DMLRuntimeException
- {
- throw new DMLRuntimeException("Unsupported operation.");
- }
-
- @Override
- public MatrixObject executeParallelMerge(int par)
- throws DMLRuntimeException
- {
- throw new DMLRuntimeException("Unsupported operation.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
deleted file mode 100644
index af87aa3..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/ResultMergeTaggedMatrixIndexes.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-
-/**
- * This class serves as composite key for the remote result merge job
- * (for any data format) in order to sort on both matrix indexes and tag
- * but group all blocks according to matrix indexes only. This prevents
- * us from doing an 2pass out-of-core algorithm at the reducer since we
- * can guarantee that the compare block (tag 0) will be the first element
- * in the iterator.
- *
- */
-public class ResultMergeTaggedMatrixIndexes implements WritableComparable<ResultMergeTaggedMatrixIndexes>
-{
-
- private MatrixIndexes _ix;
- private byte _tag = -1;
-
- public ResultMergeTaggedMatrixIndexes()
- {
- _ix = new MatrixIndexes();
- }
-
- public ResultMergeTaggedMatrixIndexes(long r, long c, byte tag)
- {
- _ix = new MatrixIndexes(r, c);
- _tag = tag;
- }
-
- public MatrixIndexes getIndexes()
- {
- return _ix;
- }
-
-
- public byte getTag()
- {
- return _tag;
- }
-
- public void setTag(byte tag)
- {
- _tag = tag;
- }
-
- @Override
- public void readFields(DataInput in)
- throws IOException
- {
- if( _ix == null )
- _ix = new MatrixIndexes();
- _ix.readFields(in);
- _tag = in.readByte();
- }
-
- @Override
- public void write(DataOutput out)
- throws IOException
- {
- _ix.write(out);
- out.writeByte(_tag);
- }
-
- @Override
- public int compareTo(ResultMergeTaggedMatrixIndexes that)
- {
- int ret = _ix.compareTo(that._ix);
-
- if( ret == 0 )
- {
- ret = ((_tag == that._tag) ? 0 :
- (_tag < that._tag)? -1 : 1);
- }
-
- return ret;
- }
-
- @Override
- public boolean equals(Object other)
- {
- if( !(other instanceof ResultMergeTaggedMatrixIndexes) )
- return false;
-
- ResultMergeTaggedMatrixIndexes that = (ResultMergeTaggedMatrixIndexes)other;
- return (_ix.equals(that._ix) && _tag == that._tag);
- }
-
- @Override
- public int hashCode() {
- throw new RuntimeException("hashCode() should never be called on instances of this class.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/Task.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/Task.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/Task.java
deleted file mode 100644
index c2ff9ac..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/Task.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.io.Serializable;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.StringTokenizer;
-
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * A task is a logical group of one or multiple iterations (each iteration is assigned to exactly one task).
- * There, each single task is executed sequentially. See TaskPartitioner for how tasks are created and
- * ParWorker for how those tasks are eventually executed.
- *
- * NOTE: (Extension possibility: group of statements)
- *
- */
-public class Task implements Serializable
-{
-
- private static final long serialVersionUID = 2815832451487164284L;
-
- public enum TaskType {
- RANGE,
- SET
- }
-
- public static final int MAX_VARNAME_SIZE = 256;
- public static final int MAX_TASK_SIZE = Integer.MAX_VALUE-1;
-
- private TaskType _type;
- private LinkedList<IntObject> _iterations; //each iteration is specified as an ordered set of index values
-
- public Task() {
- //default constructor for serialize
- }
-
- public Task( TaskType type )
- {
- _type = type;
-
- _iterations = new LinkedList<IntObject>();
- }
-
- public void addIteration( IntObject indexVal )
- {
- if( indexVal.getName().length() > MAX_VARNAME_SIZE )
- throw new RuntimeException("Cannot add iteration, MAX_VARNAME_SIZE exceeded.");
-
- if( size() >= MAX_TASK_SIZE )
- throw new RuntimeException("Cannot add iteration, MAX_TASK_SIZE reached.");
-
- _iterations.addLast( indexVal );
- }
-
- public List<IntObject> getIterations()
- {
- return _iterations;
- }
-
- public TaskType getType()
- {
- return _type;
- }
-
- public int size()
- {
- return _iterations.size();
- }
-
- /**
- *
- * @param task
- */
- public void mergeTask( Task task )
- {
- //check for set iteration type
- if( _type==TaskType.RANGE )
- throw new RuntimeException("Task Merging not supported for tasks of type ITERATION_RANGE.");
-
- //check for same iteration name
- String var1 = _iterations.getFirst().getName();
- String var2 = task._iterations.getFirst().getName();
- if( !var1.equals(var2) )
- throw new RuntimeException("Task Merging not supported for tasks with different variable names");
-
- //merge tasks
- for( IntObject o : task._iterations )
- _iterations.addLast( o );
- }
-
-
- @Override
- public String toString()
- {
- return toFormatedString();
- }
-
- /**
- *
- * @return
- */
- public String toFormatedString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("task (type=");
- sb.append(_type);
- sb.append(", iterations={");
- int count=0;
- for( IntObject dat : _iterations )
- {
- if( count!=0 )
- sb.append(";");
- sb.append("[");
- sb.append(dat.getName());
- sb.append("=");
- sb.append(dat.getLongValue());
- sb.append("]");
-
- count++;
- }
- sb.append("})");
- return sb.toString();
- }
-
- /**
- *
- * @return
- */
- public String toCompactString()
- {
- StringBuilder sb = new StringBuilder( );
- sb.append(_type);
-
- if( size() > 0 )
- {
- sb.append(".");
- IntObject dat0 = _iterations.getFirst();
- sb.append(dat0.getName());
- sb.append(".{");
-
- int count = 0;
- for( IntObject dat : _iterations )
- {
- if( count!=0 )
- sb.append(",");
- sb.append(dat.getLongValue());
- count++;
- }
-
- sb.append("}");
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @return
- */
- public String toCompactString( int maxDigits )
- {
- StringBuilder sb = new StringBuilder( );
- sb.append(_type);
-
- if( size() > 0 )
- {
- sb.append(".");
- IntObject dat0 = _iterations.getFirst();
- sb.append(dat0.getName());
- sb.append(".{");
-
- int count = 0;
- for( IntObject dat : _iterations )
- {
- if( count!=0 )
- sb.append(",");
-
- String tmp = String.valueOf(dat.getLongValue());
- for( int k=tmp.length(); k<maxDigits; k++ )
- sb.append("0");
- sb.append(tmp);
- count++;
- }
-
- sb.append("}");
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param stask
- * @return
- */
- public static Task parseCompactString( String stask )
- {
- StringTokenizer st = new StringTokenizer( stask.trim(), "." );
-
- Task newTask = new Task( TaskType.valueOf(st.nextToken()) );
- String meta = st.nextToken();
-
- //iteration data
- String sdata = st.nextToken();
- sdata = sdata.substring(1,sdata.length()-1); // remove brackets
- StringTokenizer st2 = new StringTokenizer(sdata, ",");
- while( st2.hasMoreTokens() )
- {
- //create new iteration
- String lsdata = st2.nextToken();
- IntObject ldata = new IntObject(meta,Integer.parseInt( lsdata ) );
- newTask.addIteration(ldata);
- }
-
- return newTask;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitioner.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitioner.java
deleted file mode 100644
index 0b7eb4e..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitioner.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.List;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * This is the base class for all task partitioner. For this purpose it stores relevant information such as
- * the loop specification (FROM, TO, INCR), the index variable and the task size. Furthermore, it declares two
- * prototypes: (1) full task creation, (2) streaming task creation.
- *
- * Known implementation classes: TaskPartitionerFixedsize, TaskPartitionerFactoring
- *
- */
-public abstract class TaskPartitioner
-{
-
- protected long _taskSize = -1;
-
- protected String _iterVarName = null;
- protected IntObject _fromVal = null;
- protected IntObject _toVal = null;
- protected IntObject _incrVal = null;
-
- protected long _numIter = -1;
-
-
- protected TaskPartitioner( long taskSize, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- _taskSize = taskSize;
-
- _iterVarName = iterVarName;
- _fromVal = fromVal;
- _toVal = toVal;
- _incrVal = incrVal;
-
- _numIter = (long)Math.ceil(((double)(_toVal.getLongValue()-_fromVal.getLongValue()+1 )) / _incrVal.getLongValue());
- }
-
- /**
- * Creates and returns set of all tasks for given problem at once.
- *
- * @return
- */
- public abstract List<Task> createTasks()
- throws DMLRuntimeException;
-
- /**
- * Creates set of all tasks for given problem, but streams them directly
- * into task queue. This allows for more tasks than fitting in main memory.
- *
- * @return
- */
- public abstract long createTasks( LocalTaskQueue<Task> queue )
- throws DMLRuntimeException;
-
-
- /**
- *
- * @return
- */
- public long getNumIterations()
- {
- return _numIter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
deleted file mode 100644
index cd03544..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * This factoring task partitioner virtually iterates over the given FOR loop (from, to, incr),
- * creates iterations and group them to tasks. Note that the task size is used here.
- * The tasks are created with decreasing size for good load balance of heterogeneous tasks.
- *
- *
- * See the original paper for details:
- * [Susan Flynn Hummel, Edith Schonberg, Lawrence E. Flynn:
- * Factoring: a practical and robust method for scheduling parallel loops.
- * SC 1991: 610-632]
- *
- */
-public class TaskPartitionerFactoring extends TaskPartitioner
-{
-
- private int _numThreads = -1;
-
- public TaskPartitionerFactoring( long taskSize, int numThreads, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, iterVarName, fromVal, toVal, incrVal);
-
- _numThreads = numThreads;
- }
-
- @Override
- public List<Task> createTasks()
- throws DMLRuntimeException
- {
- LinkedList<Task> tasks = new LinkedList<Task>();
-
- long lFrom = _fromVal.getLongValue();
- long lTo = _toVal.getLongValue();
- long lIncr = _incrVal.getLongValue();
-
- int P = _numThreads; // number of parallel workers
- long N = _numIter; // total number of iterations
- long R = N; // remaining number of iterations
- long K = -1; // next _numThreads task sizes
- TaskType type = null; // type of iterations: range tasks (similar to run-length encoding) make only sense if taskSize>3
-
- for( long i = lFrom; i<=lTo; )
- {
- K = determineNextBatchSize(R, P);
- R -= (K * P);
-
- type = (ParForProgramBlock.USE_RANGE_TASKS_IF_USEFUL && K>3 ) ?
- TaskType.RANGE : TaskType.SET;
-
- //for each logical processor
- for( int j=0; j<P; j++ )
- {
- if( i > lTo ) //no more iterations
- break;
-
- //create new task and add to list of tasks
- Task lTask = new Task( type );
- tasks.addLast(lTask);
-
- // add iterations to task
- if( type == TaskType.SET )
- {
- //value based tasks
- for( long k=0; k<K && i<=lTo; k++, i+=lIncr )
- {
- lTask.addIteration(new IntObject(_iterVarName, i));
- }
- }
- else
- {
- //determine end of task
- long to = Math.min( i+(K-1)*lIncr, lTo );
-
- //range based tasks
- lTask.addIteration(new IntObject(_iterVarName, i)); //from
- lTask.addIteration(new IntObject(_iterVarName, to)); //to
- lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment
-
- i = to + lIncr;
- }
- }
- }
-
- return tasks;
- }
-
- @Override
- public long createTasks(LocalTaskQueue<Task> queue)
- throws DMLRuntimeException
- {
- long numCreatedTasks = 0;
-
- long lFrom = _fromVal.getLongValue();
- long lTo = _toVal.getLongValue();
- long lIncr = _incrVal.getLongValue();
-
- int P = _numThreads; // number of parallel workers
- long N = _numIter; // total number of iterations
- long R = N; // remaining number of iterations
- long K = -1; //next _numThreads task sizes
- TaskType type = null; // type of iterations: range tasks (similar to run-length encoding) make only sense if taskSize>3
-
- try
- {
- for( long i = lFrom; i<=lTo; )
- {
- K = determineNextBatchSize(R, P);
- R -= (K * P);
-
- type = (ParForProgramBlock.USE_RANGE_TASKS_IF_USEFUL && K>3 ) ?
- TaskType.RANGE : TaskType.SET;
-
- //for each logical processor
- for( int j=0; j<P; j++ )
- {
- if( i > lTo ) //no more iterations
- break;
-
- //create new task and add to list of tasks
- Task lTask = new Task( type );
-
- // add iterations to task
- if( type == TaskType.SET )
- {
- //value based tasks
- for( long k=0; k<K && i<=lTo; k++, i+=lIncr )
- {
- lTask.addIteration(new IntObject(_iterVarName, i));
- }
- }
- else
- {
- //determine end of task
- long to = Math.min( i+(K-1)*lIncr, lTo );
-
- //range based tasks
- lTask.addIteration(new IntObject(_iterVarName, i)); //from
- lTask.addIteration(new IntObject(_iterVarName, to)); //to
- lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment
-
- i = to + lIncr;
- }
-
- //add task to queue (after all iteration added for preventing raise conditions)
- queue.enqueueTask( lTask );
- numCreatedTasks++;
- }
- }
-
- // mark end of task input stream
- queue.closeInput();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- return numCreatedTasks;
- }
-
-
- /**
- * Computes the task size (number of iterations per task) for the next numThreads tasks
- * given the number of remaining iterations R, and the number of Threads.
- *
- * NOTE: x can be set to different values, but the original paper argues for x=2.
- *
- * @param R
- * @return
- */
- protected long determineNextBatchSize(long R, int P)
- {
- int x = 2;
- long K = (long) Math.ceil((double)R / ( x * P )); //NOTE: round creates more tasks
-
- if( K < 1 ) //account for rounding errors
- K = 1;
-
- return K;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
deleted file mode 100644
index 914f387..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmax.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * Factoring with maximum constraint (e.g., if LIX matrix out-of-core and we need
- * to bound the maximum number of iterations per map task -> memory bounds)
- */
-public class TaskPartitionerFactoringCmax extends TaskPartitionerFactoring
-{
-
- protected long _constraint = -1;
-
- public TaskPartitionerFactoringCmax( long taskSize, int numThreads, long constraint, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, numThreads, iterVarName, fromVal, toVal, incrVal);
-
- _constraint = constraint;
- }
-
- @Override
- protected long determineNextBatchSize(long R, int P)
- {
- int x = 2;
- long K = (long)Math.ceil((double)R / ( x * P )); //NOTE: round creates more tasks
-
- if( K > _constraint ) //account for rounding errors
- K = _constraint;
-
- return K;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
deleted file mode 100644
index 0d1c022..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFactoringCmin.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * Factoring with minimum constraint (e.g., if communication is expensive)
- */
-public class TaskPartitionerFactoringCmin extends TaskPartitionerFactoring
-{
-
- protected long _constraint = -1;
-
- public TaskPartitionerFactoringCmin( long taskSize, int numThreads, long constraint, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, numThreads, iterVarName, fromVal, toVal, incrVal);
-
- _constraint = constraint;
- }
-
- @Override
- protected long determineNextBatchSize(long R, int P)
- {
- int x = 2;
- long K = (long) Math.ceil((double)R / ( x * P )); //NOTE: round creates more tasks
-
- if( K < _constraint ) //account for rounding errors
- K = _constraint;
-
- return K;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
deleted file mode 100644
index 54fc72a..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.Task.TaskType;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * This naive task partitioner virtually iterates over the given FOR loop (from, to, incr),
- * creates iterations and group them to tasks according to the given task size. There, all
- * tasks are equally sized.
- *
- */
-public class TaskPartitionerFixedsize extends TaskPartitioner
-{
-
- protected int _firstnPlus1 = 0; //add one to these firstn tasks
-
- public TaskPartitionerFixedsize( long taskSize, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, iterVarName, fromVal, toVal, incrVal);
- }
-
- @Override
- public List<Task> createTasks()
- throws DMLRuntimeException
- {
- LinkedList<Task> tasks = new LinkedList<Task>();
-
- //range tasks (similar to run-length encoding) make only sense if taskSize>3
- TaskType type = (ParForProgramBlock.USE_RANGE_TASKS_IF_USEFUL && _taskSize>3 ) ?
- TaskType.RANGE : TaskType.SET;
-
- long lFrom = _fromVal.getLongValue();
- long lTo = _toVal.getLongValue();
- long lIncr = _incrVal.getLongValue();
- long lfnp1 = _firstnPlus1;
-
- for( long i = lFrom; i<=lTo; )
- {
- //create new task and add to list of tasks
- Task lTask = new Task( type );
- tasks.addLast(lTask);
-
- int corr = (lfnp1-- > 0)? 1:0; //correction for static partitioner
-
- // add <tasksize> iterations to task
- // (last task might have less)
- if( type == TaskType.SET )
- {
- //value based tasks
- for( long j=0; j<_taskSize+corr && i<=lTo; j++, i+=lIncr )
- {
- lTask.addIteration(new IntObject(_iterVarName, i));
- }
- }
- else
- {
- //determine end of task
- long to = Math.min( i+(_taskSize-1+corr)*lIncr, lTo );
-
- //range based tasks
- lTask.addIteration(new IntObject(_iterVarName, i)); //from
- lTask.addIteration(new IntObject(_iterVarName, to)); //to
- lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment
-
- i = to + lIncr;
- }
- }
-
- return tasks;
- }
-
- @Override
- public long createTasks(LocalTaskQueue<Task> queue)
- throws DMLRuntimeException
- {
- long numCreatedTasks=0;
-
- //range tasks (similar to run-length encoding) make only sense if taskSize>3
- TaskType type = (ParForProgramBlock.USE_RANGE_TASKS_IF_USEFUL && _taskSize>3 ) ?
- TaskType.RANGE : TaskType.SET;
-
- long lFrom = _fromVal.getLongValue();
- long lTo = _toVal.getLongValue();
- long lIncr = _incrVal.getLongValue();
- long lfnp1 = _firstnPlus1;
-
- try
- {
- for( long i = lFrom; i<=lTo; )
- {
- //create new task and add to list of tasks
- Task lTask = new Task( type );
-
- int corr = (lfnp1-- > 0)? 1:0; //correction for static partitioner
-
- // add <tasksize> iterations to task
- // (last task might have less)
- if( type == TaskType.SET )
- {
- //value based tasks
- for( long j=0; j<_taskSize+corr && i<=lTo; j++, i+=lIncr )
- {
- lTask.addIteration(new IntObject(_iterVarName, i));
- }
- }
- else
- {
- //determine end of task
- long to = Math.min( i+(_taskSize-1+corr)*lIncr, lTo );
-
- //range based tasks
- lTask.addIteration(new IntObject(_iterVarName, i)); //from
- lTask.addIteration(new IntObject(_iterVarName, to)); //to
- lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment
-
- i = to + lIncr;
- }
-
- //add task to queue (after all iteration added for preventing raise conditions)
- queue.enqueueTask( lTask );
- numCreatedTasks++;
- }
-
- // mark end of task input stream
- queue.closeInput();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- return numCreatedTasks;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerNaive.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
deleted file mode 100644
index 2ad4806..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerNaive.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * This static task partitioner virtually iterates over the given FOR loop (from, to, incr),
- * creates iterations and group them to tasks according to a task size of numIterations/numWorkers.
- * There, all tasks are equally sized.
- *
- */
-public class TaskPartitionerNaive extends TaskPartitionerFixedsize
-{
-
- public TaskPartitionerNaive( long taskSize, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, iterVarName, fromVal, toVal, incrVal);
-
- //compute the new task size
- _taskSize = 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerStatic.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
deleted file mode 100644
index 20be635..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/TaskPartitionerStatic.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor;
-
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-
-/**
- * This static task partitioner virtually iterates over the given FOR loop (from, to, incr),
- * creates iterations and group them to tasks according to a task size of numIterations/numWorkers.
- * There, all tasks are equally sized.
- *
- */
-public class TaskPartitionerStatic extends TaskPartitionerFixedsize
-{
-
- public TaskPartitionerStatic( long taskSize, int numThreads, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal )
- {
- super(taskSize, iterVarName, fromVal, toVal, incrVal);
-
- _taskSize = _numIter / numThreads;
- _firstnPlus1 = (int)_numIter % numThreads;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
deleted file mode 100644
index 7b3423a..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MetaData;
-
-/**
- * Merged MR Job instruction to hold the actually merged instruction as well as offsets of
- * result indexes in order to split result meta data after successful execution.
- *
- */
-public class MergedMRJobInstruction
-{
-
- protected MRJobInstruction inst;
- protected LinkedList<Long> ids;
- protected HashMap<Long,Integer> outIxOffs;
- protected HashMap<Long,Integer> outIxLens;
-
- public MergedMRJobInstruction()
- {
- ids = new LinkedList<Long>();
- outIxOffs = new HashMap<Long,Integer>();
- outIxLens = new HashMap<Long,Integer>();
- }
-
- public void addInstructionMetaData(long instID, int outIxOffset, int outIxLen)
- {
- ids.add(instID);
- outIxOffs.put(instID, outIxOffset);
- outIxLens.put(instID, outIxLen);
- }
-
- /**
- *
- * @param instID
- * @param allRet
- * @return
- */
- public JobReturn constructJobReturn( long instID, JobReturn retAll )
- {
- //get output offset and len
- int off = outIxOffs.get(instID);
- int len = outIxLens.get(instID);
-
- //create partial output meta data
- JobReturn ret = new JobReturn();
- ret.successful = retAll.successful;
- if( ret.successful ) {
- ret.metadata = new MetaData[len];
- System.arraycopy(retAll.metadata, off, ret.metadata, 0, len);
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
deleted file mode 100644
index f76f515..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.Timing;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-
-/**
- *
- *
- */
-public abstract class PiggybackingWorker extends Thread
-{
-
-
- protected static final Log LOG = LogFactory.getLog(PiggybackingWorker.class.getName());
-
- protected HashMap<Long, JobReturn> _results = null;
- protected boolean _stop;
-
- protected PiggybackingWorker()
- {
- _results = new HashMap<Long, JobReturn>();
- _stop = false;
- }
-
- /**
- *
- */
- public void setStopped()
- {
- _stop = true;
- }
-
- /**
- *
- * @param instID
- * @return
- * @throws InterruptedException
- */
- public synchronized JobReturn getJobResult( long instID )
- throws InterruptedException
- {
- JobReturn ret = null;
-
- while( ret == null )
- {
- //wait for new results
- wait();
-
- //obtain job return (if available)
- ret = _results.remove( instID );
- }
-
- return ret;
- }
-
-
- /**
- *
- * @param ids
- * @param results
- */
- protected synchronized void putJobResults( LinkedList<Long> ids, LinkedList<JobReturn> results )
- {
- //make job returns available
- for( int i=0; i<ids.size(); i++ )
- _results.put(ids.get(i), results.get(i));
-
- //notify all waiting threads
- notifyAll();
- }
-
- /**
- *
- * @param workingSet
- * @return
- * @throws IllegalAccessException
- */
- protected LinkedList<MergedMRJobInstruction> mergeMRJobInstructions( LinkedList<Pair<Long,MRJobInstruction>> workingSet )
- throws IllegalAccessException
- {
- LinkedList<MergedMRJobInstruction> ret = new LinkedList<MergedMRJobInstruction>();
- Timing time = new Timing(true);
-
- //NOTE currently all merged into one (might be invalid due to memory constraints)
- MergedMRJobInstruction minst = new MergedMRJobInstruction();
- for( Pair<Long,MRJobInstruction> inst : workingSet )
- {
- long instID = inst.getKey();
- MRJobInstruction instVal = inst.getValue();
- int numOutputs = instVal.getOutputs().length;
-
- //append to current merged instruction
- if( minst.inst==null )
- {
- //deep copy first instruction
- minst.inst = new MRJobInstruction( instVal );
- minst.addInstructionMetaData( instID, 0, numOutputs );
- }
- else
- {
- //merge other instructions
- if( minst.inst.isMergableMRJobInstruction( instVal ) )
- {
- //add instruction to open merged instruction
- int offOutputs = minst.inst.getOutputs().length; //before merge
- minst.inst.mergeMRJobInstruction( instVal );
- minst.addInstructionMetaData(instID, offOutputs, numOutputs);
- }
- else
- {
- //close current merged instruction
- ret.add(minst);
- //open new merged instruction
- minst = new MergedMRJobInstruction();
- minst.inst = new MRJobInstruction( instVal );
- minst.addInstructionMetaData( instID, 0, numOutputs );
- }
- }
- }
- //close last open merged instruction
- ret.add(minst);
-
- //output log info for better understandability for users
- LOG.info("Merged MR-Job instructions: "+workingSet.size()+" --> "+ret.size()+" in "+time.stop()+"ms.");
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
deleted file mode 100644
index d385fc7..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.LinkedList;
-
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
-
-public class PiggybackingWorkerTimeSequential extends PiggybackingWorker
-{
-
- //internal configuration parameters
- private static long DEFAULT_MERGE_INTERVAL = 1000;
- private static boolean SUBSTRACT_EXEC_TIME = true;
-
- private long _time;
-
- public PiggybackingWorkerTimeSequential()
- {
- this(DEFAULT_MERGE_INTERVAL);
- }
-
- public PiggybackingWorkerTimeSequential( long timeInterval )
- {
- _time = timeInterval;
- }
-
- @Override
- public void run()
- {
- long lastTime = System.currentTimeMillis();
-
- while( !_stop )
- {
- try
- {
- // wait until next submission
- if( SUBSTRACT_EXEC_TIME ) {
- long currentTime = System.currentTimeMillis();
- if( currentTime-lastTime < _time )
- Thread.sleep( _time-(currentTime-lastTime) );
- lastTime = currentTime;
- }
- else
- Thread.sleep(_time);
-
-
- // pick job type with largest number of jobs
- LinkedList<Pair<Long,MRJobInstruction>> workingSet = RuntimePiggybacking.getMaxWorkingSet();
- if( workingSet == null )
- continue; //empty pool
-
- // merge jobs (if possible)
- LinkedList<MergedMRJobInstruction> mergedWorkingSet = mergeMRJobInstructions(workingSet);
-
- // submit all resulting jobs (currently sequential submission)
- for( MergedMRJobInstruction minst : mergedWorkingSet )
- {
- JobReturn mret = RunMRJobs.submitJob(minst.inst);
- Statistics.incrementNoOfExecutedMRJobs();
-
- // error handling
- if( !mret.successful )
- LOG.error("Failed to run merged mr-job instruction:\n"+minst.inst.toString());
-
- // split job return
- LinkedList<JobReturn> ret = new LinkedList<JobReturn>();
- for( Long id : minst.ids ){
- ret.add( minst.constructJobReturn(id, mret) );
- Statistics.decrementNoOfExecutedMRJobs();
- }
- // make job returns available and notify waiting clients
- putJobResults(minst.ids, ret);
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
deleted file mode 100644
index cad9967..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.LinkedList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- *
- * Extensions: (1) take number of running jobs into account,
- * (2) compute timeout threshold based on max and last job execution time.
- */
-public class PiggybackingWorkerUtilDecayParallel extends PiggybackingWorker
-{
-
- //internal configuration parameters
- private static long MIN_MERGE_INTERVAL = 1000;
- private static double UTILIZATION_DECAY = 0.5; //decay per minute
-
- //thread pool for parallel submit
- private ExecutorService _parSubmit = null;
-
- private long _minTime = -1;
- private double _utilDecay = -1;
- private int _par = -1;
-
- public PiggybackingWorkerUtilDecayParallel(int par)
- {
- this( MIN_MERGE_INTERVAL,
- UTILIZATION_DECAY,
- par );
- }
-
- public PiggybackingWorkerUtilDecayParallel( long minInterval, double utilDecay, int par )
- {
- _minTime = minInterval;
- _utilDecay = utilDecay;
- _par = par;
-
- //init thread pool
- _parSubmit = Executors.newFixedThreadPool(_par);
- }
-
- @Override
- public void setStopped()
- {
- //parent logic
- super.setStopped();
-
- //explicitly stop the thread pool
- _parSubmit.shutdown();
- }
-
- @Override
- public void run()
- {
- long lastTime = System.currentTimeMillis();
-
- while( !_stop )
- {
- try
- {
- long currentTime = System.currentTimeMillis()+1; //ensure > lastTime
-
- // wait until next submission
- Thread.sleep(_minTime); //wait at least minTime
-
- //continue if (prevent cluster status requests)
- if( RuntimePiggybacking.isEmptyJobPool() )
- continue;
-
- double util = RuntimePiggybackingUtils.getCurrentClusterUtilization();
- double utilThreshold = 1-Math.pow(_utilDecay, Math.ceil(((double)currentTime-lastTime)/60000));
-
- //continue to collect jobs if cluster util too high (decay to prevent starvation)
- if( util > utilThreshold ) { //cluster utilization condition
- continue; //1min - >50%, 2min - >75%, 3min - >87.5%, 4min - > 93.7%
- }
-
- // pick job type with largest number of jobs
- LinkedList<Pair<Long,MRJobInstruction>> workingSet = RuntimePiggybacking.getMaxWorkingSet();
- if( workingSet == null )
- continue; //empty pool
-
- // merge jobs (if possible)
- LinkedList<MergedMRJobInstruction> mergedWorkingSet = mergeMRJobInstructions(workingSet);
-
- // submit all resulting jobs (parallel submission)
- for( MergedMRJobInstruction minst : mergedWorkingSet )
- {
- //submit job and return results if finished
- _parSubmit.execute(new MRJobSubmitTask(minst));
- }
-
- lastTime = currentTime;
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
- }
- }
-
-
- /**
- *
- *
- */
- public class MRJobSubmitTask implements Runnable
- {
- private MergedMRJobInstruction _minst = null;
-
- public MRJobSubmitTask( MergedMRJobInstruction minst )
- {
- _minst = minst;
- }
-
- @Override
- public void run()
- {
- try
- {
- // submit mr job
- JobReturn mret = RunMRJobs.submitJob(_minst.inst);
- Statistics.incrementNoOfExecutedMRJobs();
-
- // error handling
- if( !mret.successful )
- LOG.error("Failed to run merged mr-job instruction:\n"+_minst.inst.toString());
-
- // split job return
- LinkedList<JobReturn> ret = new LinkedList<JobReturn>();
- for( Long id : _minst.ids ){
- ret.add( _minst.constructJobReturn(id, mret) );
- Statistics.decrementNoOfExecutedMRJobs();
- }
- putJobResults(_minst.ids, ret);
- }
- catch(Exception ex)
- {
- //log error and merged instruction
- LOG.error("Failed to run merged mr-job instruction:\n"+_minst.inst.toString(),ex);
-
- //handle unsuccessful job returns for failed job
- //(otherwise clients would literally wait forever for results)
- LinkedList<JobReturn> ret = new LinkedList<JobReturn>();
- for( Long id : _minst.ids ){
- JobReturn fret = new JobReturn(new MatrixCharacteristics[_minst.outIxLens.get(id)], false);
- ret.add( _minst.constructJobReturn(id, fret) );
- Statistics.decrementNoOfExecutedMRJobs();
- }
- // make job returns available and notify waiting clients
- putJobResults(_minst.ids, ret);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
deleted file mode 100644
index 9c89532..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram.parfor.mqo;
-
-import java.util.LinkedList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import com.ibm.bi.dml.lops.runtime.RunMRJobs;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.data.Pair;
-import com.ibm.bi.dml.utils.Statistics;
-
-/**
- *
- * Extensions: (1) take number of running jobs into account,
- * (2) compute timeout threshold based on max and last job execution time.
- */
-public class PiggybackingWorkerUtilTimeParallel extends PiggybackingWorker
-{
-
- //internal configuration parameters
- private static long MIN_MERGE_INTERVAL = 1000;
- private static long MAX_MERGE_INTERVAL = 60000;
- private static double UTILIZATION_THRESHOLD = 0.4; //60% occupied map tasks
-
- //thread pool for parallel submit
- private ExecutorService _parSubmit = null;
-
- private long _minTime = -1;
- private long _maxTime = -1;
- private double _utilThreshold = -1;
- private int _par = -1;
-
- public PiggybackingWorkerUtilTimeParallel(int par)
- {
- this( MIN_MERGE_INTERVAL,
- MAX_MERGE_INTERVAL,
- UTILIZATION_THRESHOLD,
- par );
- }
-
- public PiggybackingWorkerUtilTimeParallel( long minInterval, long maxInterval, double utilThreshold, int par )
- {
- _minTime = minInterval;
- _maxTime = maxInterval;
- _utilThreshold = utilThreshold;
- _par = par;
-
- //init thread pool
- _parSubmit = Executors.newFixedThreadPool(_par);
- }
-
- @Override
- public void setStopped()
- {
- //parent logic
- super.setStopped();
-
- //explicitly stop the thread pool
- _parSubmit.shutdown();
- }
-
- @Override
- public void run()
- {
- long lastTime = System.currentTimeMillis();
-
- while( !_stop )
- {
- try
- {
- long currentTime = System.currentTimeMillis();
-
- // wait until next submission
- Thread.sleep(_minTime); //wait at least minTime
- if( RuntimePiggybacking.isEmptyJobPool() )
- continue;
- double util = RuntimePiggybackingUtils.getCurrentClusterUtilization();
- if( util > _utilThreshold //cluster utilization condition
- && currentTime-lastTime<_maxTime ) //timeout condition
- {
- continue;
- }
-
- // pick job type with largest number of jobs
- LinkedList<Pair<Long,MRJobInstruction>> workingSet = RuntimePiggybacking.getMaxWorkingSet();
- if( workingSet == null )
- continue; //empty pool
-
- // merge jobs (if possible)
- LinkedList<MergedMRJobInstruction> mergedWorkingSet = mergeMRJobInstructions(workingSet);
-
- // submit all resulting jobs (parallel submission)
- for( MergedMRJobInstruction minst : mergedWorkingSet )
- {
- //submit job and return results if finished
- _parSubmit.execute(new MRJobSubmitTask(minst));
- }
-
- lastTime = currentTime;
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
- }
- }
-
-
- /**
- *
- *
- */
- public class MRJobSubmitTask implements Runnable
- {
- private MergedMRJobInstruction _minst = null;
-
- public MRJobSubmitTask( MergedMRJobInstruction minst )
- {
- _minst = minst;
- }
-
- @Override
- public void run()
- {
- try
- {
- // submit mr job
- JobReturn mret = RunMRJobs.submitJob(_minst.inst);
- Statistics.incrementNoOfExecutedMRJobs();
-
- // error handling
- if( !mret.successful )
- LOG.error("Failed to run merged mr-job instruction:\n"+_minst.inst.toString());
-
- // split job return
- LinkedList<JobReturn> ret = new LinkedList<JobReturn>();
- for( Long id : _minst.ids ){
- ret.add( _minst.constructJobReturn(id, mret) );
- Statistics.decrementNoOfExecutedMRJobs();
- }
- // make job returns available and notify waiting clients
- putJobResults(_minst.ids, ret);
- }
- catch(Exception ex)
- {
- throw new RuntimeException(ex);
- }
- }
-
- }
-}
[54/78] [abbrv] incubator-systemml git commit: Update source
references based on new package structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/NegationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/NegationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/NegationTest.java
index f2f1a94..92b5913 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/NegationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/NegationTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/PrintTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/PrintTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/PrintTest.java
index f9f56bc..d018a17 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/PrintTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/PrintTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class PrintTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/QRSolverTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/QRSolverTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/QRSolverTest.java
index dfade92..0b45f68 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/QRSolverTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/QRSolverTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class QRSolverTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptyTest.java
index afe9fb3..d61076e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptyTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptyTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.ParameterizedBuiltinOp;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
public class RemoveEmptyTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ReplaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ReplaceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ReplaceTest.java
index b679a20..8944cb1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ReplaceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/ReplaceTest.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import java.util.HashMap;
import java.util.Random;
@@ -23,14 +23,14 @@ import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
-import com.ibm.bi.dml.utils.Statistics;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
public class ReplaceTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RoundTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RoundTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RoundTest.java
index 1ccd7b5..62223bf 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RoundTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RoundTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SinTest.java
index a984e0a..fc2bd33 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SinTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SqrtTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SqrtTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SqrtTest.java
index 173c382..a9f6e73 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SqrtTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SqrtTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TanTest.java
index 18eead2..ef7fd5c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TanTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TransposeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TransposeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TransposeTest.java
index 9628f5a..8ca9bc1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TransposeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/TransposeTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.matrix;
+package org.apache.sysml.test.integration.functions.unary.matrix;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ACosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ACosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ACosTest.java
index 1f4dc78..4d52bfb 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ACosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ACosTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ASinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ASinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ASinTest.java
index e7818b1..0d64b55 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ASinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ASinTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ATanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ATanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ATanTest.java
index 81d6307..ad343f7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ATanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ATanTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/AbsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/AbsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/AbsTest.java
index 3f0af61..af38f61 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/AbsTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/AbsTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/BooleanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/BooleanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/BooleanTest.java
index b20514a..1bca029 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/BooleanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/BooleanTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/CosTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/CosTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/CosTest.java
index d948772..142746a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/CosTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/CosTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ExponentTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ExponentTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ExponentTest.java
index 9885850..02e413a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ExponentTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/ExponentTest.java
@@ -15,14 +15,14 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLException;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/FullDistributionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/FullDistributionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/FullDistributionTest.java
index d7b875b..3561bbd 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/FullDistributionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/FullDistributionTest.java
@@ -15,17 +15,17 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import java.util.HashMap;
import java.util.Random;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class FullDistributionTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NegationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NegationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NegationTest.java
index 9e9173e..6022de3 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NegationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NegationTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NotTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NotTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NotTest.java
index 711137b..982d0cc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NotTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/NotTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/PrintTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/PrintTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/PrintTest.java
index 4578fdf..6c5743a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/PrintTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/PrintTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/RoundTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/RoundTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/RoundTest.java
index f5695f5..7606747 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/RoundTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/RoundTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
public class RoundTest extends AutomatedTestBase
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SinTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SinTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SinTest.java
index 4d394ba..c0c6773 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SinTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SinTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SqrtTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SqrtTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SqrtTest.java
index 5698211..87bce4c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SqrtTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/SqrtTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest.java
index 9a3df23..37a5b48 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest2.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest2.java
index c55e1f5..40d1f28 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest2.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTest2.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTestCtrlStr.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTestCtrlStr.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTestCtrlStr.java
index f0f0796..13a59a4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTestCtrlStr.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/StopTestCtrlStr.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/TanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/TanTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/TanTest.java
index ced2bb7..f35b195 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/TanTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/scalar/TanTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.unary.scalar;
+package org.apache.sysml.test.integration.functions.unary.scalar;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/functions/vect/AutoVectorizationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/vect/AutoVectorizationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/vect/AutoVectorizationTest.java
index 6d94677..1b99d6c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/vect/AutoVectorizationTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/vect/AutoVectorizationTest.java
@@ -15,16 +15,16 @@
*
*/
-package com.ibm.bi.dml.test.integration.functions.vect;
+package org.apache.sysml.test.integration.functions.vect;
import java.util.HashMap;
import org.junit.Test;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.test.integration.AutomatedTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
index 1c3fa9c..77efbdb 100644
--- a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
@@ -15,13 +15,13 @@
*
*/
-package com.ibm.bi.dml.test.integration.scalability;
+package org.apache.sysml.test.integration.scalability;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedScalabilityTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
-import com.ibm.bi.dml.test.utils.TestUtils;
+import org.apache.sysml.test.integration.AutomatedScalabilityTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
index eac13b6..a8ee91c 100644
--- a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
@@ -15,12 +15,12 @@
*
*/
-package com.ibm.bi.dml.test.integration.scalability;
+package org.apache.sysml.test.integration.scalability;
import org.junit.Test;
-import com.ibm.bi.dml.test.integration.AutomatedScalabilityTestBase;
-import com.ibm.bi.dml.test.integration.TestConfiguration;
+import org.apache.sysml.test.integration.AutomatedScalabilityTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index d6f0308..c3a9295 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -15,7 +15,7 @@
*
*/
-package com.ibm.bi.dml.test.utils;
+package org.apache.sysml.test.utils;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
@@ -52,14 +52,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
-import com.ibm.bi.dml.runtime.io.IOUtilFunctions;
-import com.ibm.bi.dml.runtime.matrix.data.IJV;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixCell;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex;
-import com.ibm.bi.dml.runtime.matrix.data.SparseRowsIterator;
-import com.ibm.bi.dml.test.integration.BinaryMatrixCharacteristics;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixCell;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
+import org.apache.sysml.test.integration.BinaryMatrixCharacteristics;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/applications/ctableStats/stratstats.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/applications/ctableStats/stratstats.dml b/src/test/scripts/applications/ctableStats/stratstats.dml
index ee35422..ea3eda4 100644
--- a/src/test/scripts/applications/ctableStats/stratstats.dml
+++ b/src/test/scripts/applications/ctableStats/stratstats.dml
@@ -307,7 +307,7 @@ print ("END STRATIFIED STATISTICS SCRIPT");
deNaN = externalFunction (Matrix[Double] A) return (Matrix[Double] B)
- implemented in (classname = "com.ibm.bi.dml.udf.lib.DeNaNWrapper", exectype = "mem");
+ implemented in (classname = "org.apache.sysml.udf.lib.DeNaNWrapper", exectype = "mem");
fStat_tailprob = function (Matrix[double] fStat, Matrix[double] df_1, Matrix[double] df_2) return (Matrix[double] tailprob)
{ # TEMPORARY IMPLEMENTATION
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/ColSumScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColSumScalarTest.dml b/src/test/scripts/functions/aggregate/ColSumScalarTest.dml
index 9a04652..a507629 100644
--- a/src/test/scripts/functions/aggregate/ColSumScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/ColSumScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.ColSumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.ColSumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/ColSumTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColSumTest.dml b/src/test/scripts/functions/aggregate/ColSumTest.dml
index b17e8cb..aa479af 100644
--- a/src/test/scripts/functions/aggregate/ColSumTest.dml
+++ b/src/test/scripts/functions/aggregate/ColSumTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.ColSumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.ColSumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/LengthScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/LengthScalarTest.dml b/src/test/scripts/functions/aggregate/LengthScalarTest.dml
index 4bd233f..d4be7d1 100644
--- a/src/test/scripts/functions/aggregate/LengthScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/LengthScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/LengthTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/LengthTest.dml b/src/test/scripts/functions/aggregate/LengthTest.dml
index ba771a8..d542f5e 100644
--- a/src/test/scripts/functions/aggregate/LengthTest.dml
+++ b/src/test/scripts/functions/aggregate/LengthTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/MaxScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/MaxScalarTest.dml b/src/test/scripts/functions/aggregate/MaxScalarTest.dml
index fa6a0fe..126228d 100644
--- a/src/test/scripts/functions/aggregate/MaxScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/MaxScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/MaxTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/MaxTest.dml b/src/test/scripts/functions/aggregate/MaxTest.dml
index 8e318b1..bc822f7 100644
--- a/src/test/scripts/functions/aggregate/MaxTest.dml
+++ b/src/test/scripts/functions/aggregate/MaxTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.MaxTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.MaxTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/MinScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/MinScalarTest.dml b/src/test/scripts/functions/aggregate/MinScalarTest.dml
index f2aabdf..5a8c018 100644
--- a/src/test/scripts/functions/aggregate/MinScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/MinScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/MinTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/MinTest.dml b/src/test/scripts/functions/aggregate/MinTest.dml
index fdeeece..c48cc20 100644
--- a/src/test/scripts/functions/aggregate/MinTest.dml
+++ b/src/test/scripts/functions/aggregate/MinTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.MinTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.MinTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/NColScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/NColScalarTest.dml b/src/test/scripts/functions/aggregate/NColScalarTest.dml
index f58e89e..9d3d7be 100644
--- a/src/test/scripts/functions/aggregate/NColScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/NColScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/NColTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/NColTest.dml b/src/test/scripts/functions/aggregate/NColTest.dml
index 1b74270..ba8cb48 100644
--- a/src/test/scripts/functions/aggregate/NColTest.dml
+++ b/src/test/scripts/functions/aggregate/NColTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.NColTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.NColTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/NRowScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/NRowScalarTest.dml b/src/test/scripts/functions/aggregate/NRowScalarTest.dml
index 2b8cffc..d4f1137 100644
--- a/src/test/scripts/functions/aggregate/NRowScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/NRowScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/NRowTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/NRowTest.dml b/src/test/scripts/functions/aggregate/NRowTest.dml
index 0d4f3b6..c0de24d 100644
--- a/src/test/scripts/functions/aggregate/NRowTest.dml
+++ b/src/test/scripts/functions/aggregate/NRowTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.NRowTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.NRowTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/ProdScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ProdScalarTest.dml b/src/test/scripts/functions/aggregate/ProdScalarTest.dml
index 5f61108..eab3102 100644
--- a/src/test/scripts/functions/aggregate/ProdScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/ProdScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.ProdTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.ProdTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/ProdTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ProdTest.dml b/src/test/scripts/functions/aggregate/ProdTest.dml
index 688dae9..f8ede49 100644
--- a/src/test/scripts/functions/aggregate/ProdTest.dml
+++ b/src/test/scripts/functions/aggregate/ProdTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.ProdTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.ProdTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/RowSumScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowSumScalarTest.dml b/src/test/scripts/functions/aggregate/RowSumScalarTest.dml
index 8997d4b..9cbf0a4 100644
--- a/src/test/scripts/functions/aggregate/RowSumScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/RowSumScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.LengthTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.LengthTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/RowSumTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowSumTest.dml b/src/test/scripts/functions/aggregate/RowSumTest.dml
index 4206af5..b9d41c5 100644
--- a/src/test/scripts/functions/aggregate/RowSumTest.dml
+++ b/src/test/scripts/functions/aggregate/RowSumTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.RowSumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.RowSumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/SumScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/SumScalarTest.dml b/src/test/scripts/functions/aggregate/SumScalarTest.dml
index 69b4ed5..c398c36 100644
--- a/src/test/scripts/functions/aggregate/SumScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/SumScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.SumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.SumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/SumTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/SumTest.dml b/src/test/scripts/functions/aggregate/SumTest.dml
index de166fe..dad7881 100644
--- a/src/test/scripts/functions/aggregate/SumTest.dml
+++ b/src/test/scripts/functions/aggregate/SumTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.SumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.SumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/TraceScalarTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/TraceScalarTest.dml b/src/test/scripts/functions/aggregate/TraceScalarTest.dml
index c7514bd..8a03e04 100644
--- a/src/test/scripts/functions/aggregate/TraceScalarTest.dml
+++ b/src/test/scripts/functions/aggregate/TraceScalarTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.SumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.SumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/aggregate/TraceTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/TraceTest.dml b/src/test/scripts/functions/aggregate/TraceTest.dml
index d2e5159..7cef94b 100644
--- a/src/test/scripts/functions/aggregate/TraceTest.dml
+++ b/src/test/scripts/functions/aggregate/TraceTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.aggregate.SumTest.java
+# junit test class: org.apache.sysml.test.integration.functions.aggregate.SumTest.java
$$readhelper$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/AMultASpecial1Test.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/AMultASpecial1Test.dml b/src/test/scripts/functions/binary/matrix/AMultASpecial1Test.dml
index f021e0d..77ce277 100644
--- a/src/test/scripts/functions/binary/matrix/AMultASpecial1Test.dml
+++ b/src/test/scripts/functions/binary/matrix/AMultASpecial1Test.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
A = A %*% A
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/AMultBSpecial2Test.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/AMultBSpecial2Test.dml b/src/test/scripts/functions/binary/matrix/AMultBSpecial2Test.dml
index 33c51f6..9da514e 100644
--- a/src/test/scripts/functions/binary/matrix/AMultBSpecial2Test.dml
+++ b/src/test/scripts/functions/binary/matrix/AMultBSpecial2Test.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm1.dml b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm1.dml
index 12d88ee..bc23b59 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm1.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm1.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm2.dml b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm2.dml
index 633e1a1..29944b2 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm2.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm3.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm3.dml b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm3.dml
index 515e264..5c5af30 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm3.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionMultiplicationTerm3.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseAdditionMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionTest.dml
index 84fabfc..d954e3d 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseAdditionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseAdditionTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionVariableDimensionsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionVariableDimensionsTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionVariableDimensionsTest.dml
index ef3bab9..12b7191 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseAdditionVariableDimensionsTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseAdditionVariableDimensionsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseAdditionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseAdditionTest.java
A = read("$$indir$$a", rows=$$rows1$$, cols=$$cols1$$, format="text");
B = read("$$indir$$b", rows=$$rows2$$, cols=$$cols2$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseDivisionTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionTest.dml
index df67874..7af4676 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseDivisionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseDivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseDivisionTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionVariableDimensionsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseDivisionVariableDimensionsTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionVariableDimensionsTest.dml
index 7091505..b407d6a 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseDivisionVariableDimensionsTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseDivisionVariableDimensionsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseDivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseDivisionTest.java
A = read("$$indir$$a", rows=$$rows1$$, cols=$$cols1$$, format="text");
B = read("$$indir$$b", rows=$$rows2$$, cols=$$cols2$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseModulusTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseModulusTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseModulusTest.dml
index 5189ff1..cd01227 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseModulusTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseModulusTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseModulusTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseModulusTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseModulusVariableDimensionsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseModulusVariableDimensionsTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseModulusVariableDimensionsTest.dml
index bc76a78..23f5ef9 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseModulusVariableDimensionsTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseModulusVariableDimensionsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseModulusTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseModulusTest.java
A = read("$$indir$$a", rows=$$rows1$$, cols=$$cols1$$, format="text");
B = read("$$indir$$b", rows=$$rows2$$, cols=$$cols2$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationTest.dml
index 04df73b..a2e1b2e 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationVariableDimensionsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationVariableDimensionsTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationVariableDimensionsTest.dml
index 2ef4cf5..c488f1c 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationVariableDimensionsTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseMultiplicationVariableDimensionsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseMultiplicationTest.java
A = read("$$indir$$a", rows=$$rows1$$, cols=$$cols1$$, format="text");
B = read("$$indir$$b", rows=$$rows2$$, cols=$$cols2$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionTest.dml
index 8136341..6143642 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseSubtractionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseSubtractionTest.java
A = read("$$indir$$a", rows=$$rows$$, cols=$$cols$$, format="text");
B = read("$$indir$$b", rows=$$rows$$, cols=$$cols$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionVariableDimensionsTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionVariableDimensionsTest.dml b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionVariableDimensionsTest.dml
index dddb6d2..7e6ccb4 100644
--- a/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionVariableDimensionsTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ElementwiseSubtractionVariableDimensionsTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ElementwiseSubtractionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ElementwiseSubtractionTest.java
A = read("$$indir$$a", rows=$$rows1$$, cols=$$cols1$$, format="text");
B = read("$$indir$$b", rows=$$rows2$$, cols=$$cols2$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/MatrixMultiplicationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/MatrixMultiplicationTest.dml b/src/test/scripts/functions/binary/matrix/MatrixMultiplicationTest.dml
index d2c4670..21006d9 100644
--- a/src/test/scripts/functions/binary/matrix/MatrixMultiplicationTest.dml
+++ b/src/test/scripts/functions/binary/matrix/MatrixMultiplicationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.MatrixMultiplicationTest.java
A = read("$$indir$$a", rows=$$m$$, cols=$$n1$$, format="text");
B = read("$$indir$$b", rows=$$n2$$, cols=$$k$$, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/MatrixVectorMultiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/MatrixVectorMultiplication.dml b/src/test/scripts/functions/binary/matrix/MatrixVectorMultiplication.dml
index 44b5429..fdec414 100644
--- a/src/test/scripts/functions/binary/matrix/MatrixVectorMultiplication.dml
+++ b/src/test/scripts/functions/binary/matrix/MatrixVectorMultiplication.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1);
x = read($2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ScalarAdditionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ScalarAdditionTest.dml b/src/test/scripts/functions/binary/matrix/ScalarAdditionTest.dml
index fec79f1..c63787e 100644
--- a/src/test/scripts/functions/binary/matrix/ScalarAdditionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ScalarAdditionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ScalarAdditionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ScalarAdditionTest.java
$$vardeclaration$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ScalarDivisionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ScalarDivisionTest.dml b/src/test/scripts/functions/binary/matrix/ScalarDivisionTest.dml
index 1ab7bdb..22e2dab 100644
--- a/src/test/scripts/functions/binary/matrix/ScalarDivisionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ScalarDivisionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ScalarDivisionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ScalarDivisionTest.java
$$vardeclaration$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ScalarModulusTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ScalarModulusTest.dml b/src/test/scripts/functions/binary/matrix/ScalarModulusTest.dml
index 09414fa..bdc3b53 100644
--- a/src/test/scripts/functions/binary/matrix/ScalarModulusTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ScalarModulusTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ScalarModulusTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ScalarModulusTest.java
$$vardeclaration$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ScalarMultiplicationTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ScalarMultiplicationTest.dml b/src/test/scripts/functions/binary/matrix/ScalarMultiplicationTest.dml
index 9bffebc..e5c8309 100644
--- a/src/test/scripts/functions/binary/matrix/ScalarMultiplicationTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ScalarMultiplicationTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ScalarMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ScalarMultiplicationTest.java
$$vardeclaration$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/ScalarSubtractionTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/ScalarSubtractionTest.dml b/src/test/scripts/functions/binary/matrix/ScalarSubtractionTest.dml
index e41e5b7..5af9349 100644
--- a/src/test/scripts/functions/binary/matrix/ScalarSubtractionTest.dml
+++ b/src/test/scripts/functions/binary/matrix/ScalarSubtractionTest.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.ScalarSubtractionTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.ScalarSubtractionTest.java
$$vardeclaration$$
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication.dml b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication.dml
index 8365183..6729d3e 100644
--- a/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication.dml
+++ b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$2, cols=1, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication2.dml b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication2.dml
index d555597..03983bc 100644
--- a/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication2.dml
+++ b/src/test/scripts/functions/binary/matrix/UltraSparseMatrixMultiplication2.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$2, cols=1, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Addition.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Addition.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Addition.dml
index ce19a4c..4ec8b9e 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Addition.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Addition.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$5, cols=$6, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Division.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Division.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Division.dml
index bd901c7..37bb2aa 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Division.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Division.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$5, cols=$6, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Multiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Multiplication.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Multiplication.dml
index 8067c1d..435f409 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Multiplication.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Multiplication.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$5, cols=$6, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Substraction.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Substraction.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Substraction.dml
index b1e85e7..0b0ca01 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Substraction.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixCellwiseOperation_Substraction.dml
@@ -16,7 +16,7 @@
#
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, rows=$2, cols=$3, format="text");
B = read($4, rows=$5, cols=$6, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Addition.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Addition.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Addition.dml
index 4c84fe5..42bda51 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Addition.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Addition.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Division.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Division.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Division.dml
index e675d1d..fb5222d 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Division.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Division.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Multiplication.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Multiplication.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Multiplication.dml
index 39a7db7..1dd6b94 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Multiplication.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Multiplication.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Substraction.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Substraction.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Substraction.dml
index a86f8da..ad2528a 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Substraction.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorColCellwiseOperation_Substraction.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7cd7dc2b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Addition.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Addition.dml b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Addition.dml
index 4c84fe5..42bda51 100644
--- a/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Addition.dml
+++ b/src/test/scripts/functions/binary/matrix_full_cellwise/FullMatrixVectorRowCellwiseOperation_Addition.dml
@@ -17,7 +17,7 @@
#-------------------------------------------------------------
-# junit test class: com.ibm.bi.dml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
+# junit test class: org.apache.sysml.test.integration.functions.binary.matrix.FullMatrixMultiplicationTest.java
A = read($1, format="text");
B = read($2, format="text");
[50/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/DMLException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/DMLException.java b/src/main/java/com/ibm/bi/dml/api/DMLException.java
deleted file mode 100644
index 078c067..0000000
--- a/src/main/java/com/ibm/bi/dml/api/DMLException.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api;
-
-/**
- * <p>Exception occurring in the DML framework.</p>
- */
-public class DMLException extends Exception
-{
-
- public static String ERROR_MSG_DELIMITER = " : ";
-
- private static final long serialVersionUID = 1L;
-
-
- /**
- * @see java.lang.Exception#Exception()
- */
- public DMLException() {
- super();
- }
-
- /**
- * @see java.lang.Exception#Exception(String)
- */
- public DMLException(String message) {
- super(message);
- }
-
- /**
- * @see java.lang.Exception#Exception(Throwable)
- */
- public DMLException(Throwable cause) {
- super(cause);
- }
-
- /**
- * @see java.lang.Exception#Exception(String, Throwable)
- */
- public DMLException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/DMLScript.java b/src/main/java/com/ibm/bi/dml/api/DMLScript.java
deleted file mode 100644
index 0b2d59e..0000000
--- a/src/main/java/com/ibm/bi/dml/api/DMLScript.java
+++ /dev/null
@@ -1,990 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.api;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Scanner;
-
-import javax.xml.parsers.ParserConfigurationException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.xml.sax.SAXException;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.debug.DMLDebugger;
-import com.ibm.bi.dml.debug.DMLDebuggerException;
-import com.ibm.bi.dml.debug.DMLDebuggerProgramInfo;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel;
-import com.ibm.bi.dml.hops.globalopt.GlobalOptimizerWrapper;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.AParserWrapper;
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.LanguageException;
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheStatistics;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheableData;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDHandler;
-import com.ibm.bi.dml.runtime.matrix.CleanupMR;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRConfigurationNames;
-import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Explain;
-import com.ibm.bi.dml.utils.Explain.ExplainCounts;
-import com.ibm.bi.dml.utils.Explain.ExplainType;
-import com.ibm.bi.dml.utils.Statistics;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-// import com.ibm.bi.dml.utils.visualize.DotGraph;
-import com.ibm.bi.dml.yarn.DMLYarnClientProxy;
-
-
-public class DMLScript
-{
- public enum RUNTIME_PLATFORM {
- HADOOP, // execute all matrix operations in MR
- SINGLE_NODE, // execute all matrix operations in CP
- HYBRID, // execute matrix operations in CP or MR
- HYBRID_SPARK, // execute matrix operations in CP or Spark
- SPARK // execute matrix operations in Spark
- };
-
- public static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID; //default exec mode
- public static boolean STATISTICS = false; //default statistics
- public static boolean ENABLE_DEBUG_MODE = false; //default debug mode
- public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default local spark configuration - used for local testing
- public static String DML_FILE_PATH_ANTLR_PARSER = null;
- public static ExplainType EXPLAIN = ExplainType.NONE; //default explain
-
- // flag that indicates whether or not to suppress any prints to stdout
- public static boolean _suppressPrint2Stdout = false;
-
- public static String _uuid = IDHandler.createDistributedUniqueID();
- public static boolean _activeAM = false;
-
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static String USAGE =
- "Usage is " + DMLScript.class.getCanonicalName() + " -f <filename>"
- //+ " (-exec <mode>)?" + " (-explain <type>)?" + " (-stats)?" + " (-clean)?" + " (-config=<config_filename>)?
- + " [-options] ([-args | -nvargs] <args-list>)? \n"
- + " -f: <filename> will be interpreted as a filename path (if <filename> is prefixed\n"
- + " with hdfs or gpfs it is read from DFS, otherwise from local file system)\n"
- //undocumented feature in beta 08/2014 release
- //+ " -s: <filename> will be interpreted as a DML script string \n"
- + " -python: (optional) parses Python-like DML\n"
- + " -debug: (optional) run in debug mode\n"
- // Later add optional flags to indicate optimizations turned on or off. Currently they are turned off.
- //+ " -debug: <flags> (optional) run in debug mode\n"
- //+ " Optional <flags> that is supported for this mode is optimize=(on|off)\n"
- + " -exec: <mode> (optional) execution mode (hadoop, singlenode, [hybrid], hybrid_spark)\n"
- + " -explain: <type> (optional) explain plan (hops, [runtime], recompile_hops, recompile_runtime)\n"
- + " -stats: (optional) monitor and report caching/recompilation statistics\n"
- + " -clean: (optional) cleanup all SystemML working directories (FS, DFS).\n"
- + " All other flags are ignored in this mode. \n"
- + " -config: (optional) use config file <config_filename> (default: use parameter\n"
- + " values in default SystemML-config.xml config file; if <config_filename> is\n"
- + " prefixed with hdfs or gpfs it is read from DFS, otherwise from local file system)\n"
- + " -args: (optional) parameterize DML script with contents of [args list], ALL args\n"
- + " after -args flag, each argument must be an unnamed-argument, where 1st value\n"
- + " after -args will replace $1 in DML script, 2nd value will replace $2, etc.\n"
- + " -nvargs: (optional) parameterize DML script with contents of [args list], ALL args\n"
- + " after -nvargs flag, each argument must be be named-argument of form argName=argValue,\n"
- + " where value will replace $argName in DML script, argName must be a valid DML variable\n"
- + " name (start with letter, contain only letters, numbers, or underscores).\n"
- + " <args-list>: (optional) args to DML script \n"
- + " -? | -help: (optional) show this help message \n";
-
-
- ///////////////////////////////
- // public external interface
- ////////
-
- public static String getUUID() {
- return _uuid;
- }
-
- /**
- * Used to set master UUID on all nodes (in parfor remote_mr, where DMLScript passed)
- * in order to simplify cleanup of scratch_space and local working dirs.
- *
- * @param uuid
- */
- public static void setUUID(String uuid)
- {
- _uuid = uuid;
- }
-
- public static boolean suppressPrint2Stdout() {
- return _suppressPrint2Stdout;
- }
-
- public static void setActiveAM(){
- _activeAM = true;
- }
-
- public static boolean isActiveAM(){
- return _activeAM;
- }
-
-
- /**
- * Default DML script invocation (e.g., via 'hadoop jar SystemML.jar -f Test.dml')
- *
- * @param args
- * @throws ParseException
- * @throws IOException
- * @throws SAXException
- * @throws ParserConfigurationException
- */
- public static void main(String[] args)
- throws IOException, DMLException
- {
- Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf());
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-
- try {
- DMLScript.executeScript(conf, otherArgs);
- } catch (DMLScriptException e){
- // In case of DMLScriptException, simply print the error message.
- System.err.println(e.getMessage());
- }
- }
-
- public static boolean executeScript( String[] args )
- throws DMLException
- {
- Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf());
- return executeScript( conf, args );
- }
-
- /**
- * This version of executeScript() is invoked from RJaqlUdf (from BigR).
- *
- * @param conf
- * @param args
- * @param suppress
- * @return
- * @throws DMLException
- */
- public static String executeScript( Configuration conf, String[] args, boolean suppress)
- throws DMLException
- {
- _suppressPrint2Stdout = suppress;
- try {
- boolean ret = executeScript(conf, args);
- return Boolean.toString(ret);
- } catch(DMLScriptException e) {
- return e.getMessage();
- }
- }
-
- /**
- * Single entry point for all public invocation alternatives (e.g.,
- * main, executeScript, JaqlUdf etc)
- *
- * @param conf
- * @param args
- * @return
- * @throws LanguageException
- */
- public static boolean executeScript( Configuration conf, String[] args )
- throws DMLException
- {
- boolean ret = false;
-
- //Step 1: parse arguments
- //check for help
- if( args.length==0 || (args.length==1 && (args[0].equalsIgnoreCase("-help")|| args[0].equalsIgnoreCase("-?"))) ){
- System.err.println( USAGE );
- return true;
- }
-
- //check for clean
- else if( args.length==1 && args[0].equalsIgnoreCase("-clean") ){
- cleanSystemMLWorkspace();
- return true;
- }
-
- //check number of args - print usage if incorrect
- if( args.length < 2 ){
- System.err.println( "ERROR: Unrecognized invocation arguments." );
- System.err.println( USAGE );
- return ret;
- }
-
- //check script arg - print usage if incorrect
- if (!(args[0].equals("-f") || args[0].equals("-s"))){
- System.err.println("ERROR: First argument must be either -f or -s");
- System.err.println( USAGE );
- return ret;
- }
-
- //parse arguments and set execution properties
- RUNTIME_PLATFORM oldrtplatform = rtplatform; //keep old rtplatform
- ExplainType oldexplain = EXPLAIN; //keep old explain
-
- // Reset global flags to avoid errors in test suite
- ENABLE_DEBUG_MODE = false;
-
- boolean parsePyDML = false;
- try
- {
- String fnameOptConfig = null; //optional config filename
- String[] scriptArgs = null; //optional script arguments
- boolean namedScriptArgs = false;
-
- for( int i=2; i<args.length; i++ )
- {
- if( args[i].equalsIgnoreCase("-explain") ) {
- EXPLAIN = ExplainType.RUNTIME;
- if( args.length > (i+1) && !args[i+1].startsWith("-") )
- EXPLAIN = Explain.parseExplainType(args[++i]);
- }
- else if( args[i].equalsIgnoreCase("-stats") )
- STATISTICS = true;
- else if ( args[i].equalsIgnoreCase("-exec")) {
- rtplatform = parseRuntimePlatform(args[++i]);
- if( rtplatform==null )
- return ret;
- }
- else if (args[i].startsWith("-config="))
- fnameOptConfig = args[i].substring(8).replaceAll("\"", "");
- else if( args[i].equalsIgnoreCase("-debug") ) {
- ENABLE_DEBUG_MODE = true;
- }
- else if( args[i].equalsIgnoreCase("-python") ) {
- parsePyDML = true;
- }
- else if (args[i].startsWith("-args") || args[i].startsWith("-nvargs")) {
- namedScriptArgs = args[i].startsWith("-nvargs"); i++;
- scriptArgs = new String[args.length - i];
- System.arraycopy(args, i, scriptArgs, 0, scriptArgs.length);
- break;
- }
- else{
- System.err.println("ERROR: Unknown argument: " + args[i]);
- return ret;
- }
- }
-
- //set log level
- if (!ENABLE_DEBUG_MODE)
- setLoggingProperties( conf );
-
- //Step 2: prepare script invocation
- String dmlScriptStr = readDMLScript(args[0], args[1]);
- HashMap<String, String> argVals = createArgumentsMap(namedScriptArgs, scriptArgs);
-
- DML_FILE_PATH_ANTLR_PARSER = args[1];
-
- //Step 3: invoke dml script
- printInvocationInfo(args[1], fnameOptConfig, argVals);
- if (ENABLE_DEBUG_MODE) {
- // inner try loop is just to isolate the debug exception, which will allow to manage the bugs from debugger v/s runtime
- launchDebugger(dmlScriptStr, fnameOptConfig, argVals, parsePyDML);
- }
- else {
- execute(dmlScriptStr, fnameOptConfig, argVals, args, parsePyDML);
- }
-
- ret = true;
- }
- catch (DMLScriptException e) {
- //rethrow DMLScriptException to propagate stop call
- throw e;
- }
- catch(Exception ex)
- {
- LOG.error("Failed to execute DML script.", ex);
- throw new DMLException(ex);
- }
- finally
- {
- //reset runtime platform and visualize flag
- rtplatform = oldrtplatform;
- EXPLAIN = oldexplain;
- }
-
- return ret;
- }
-
- ///////////////////////////////
- // private internal utils (argument parsing)
- ////////
-
- /**
- *
- * @param hasNamedArgs
- * @param scriptArguments
- * @throws LanguageException
- */
- protected static HashMap<String,String> createArgumentsMap(boolean hasNamedArgs, String[] args)
- throws LanguageException
- {
- HashMap<String, String> argMap = new HashMap<String,String>();
-
- if (args == null)
- return argMap;
-
- for(int i=1; i<=args.length; i++)
- {
- String arg = args[i-1];
-
- if (arg.equalsIgnoreCase("-l") || arg.equalsIgnoreCase("-log") ||
- arg.equalsIgnoreCase("-v") || arg.equalsIgnoreCase("-visualize")||
- arg.equalsIgnoreCase("-explain") ||
- arg.equalsIgnoreCase("-debug") ||
- arg.equalsIgnoreCase("-stats") ||
- arg.equalsIgnoreCase("-exec") ||
- arg.equalsIgnoreCase("-debug") ||
- arg.startsWith("-config="))
- {
- throw new LanguageException("-args or -nvargs must be the final argument for DMLScript!");
- }
-
- //parse arguments (named args / args by position)
- if(hasNamedArgs)
- {
- // CASE: named argument argName=argValue -- must add <argName, argValue> pair to _argVals
- String[] argPieces = arg.split("=");
- if(argPieces.length < 2)
- throw new LanguageException("for -nvargs option, elements in arg list must be named and have form argName=argValue");
- String argName = argPieces[0];
- StringBuilder sb = new StringBuilder();
- for (int jj=1; jj < argPieces.length; jj++){
- sb.append(argPieces[jj]);
- }
-
- String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$";
- if (!argName.matches(varNameRegex))
- throw new LanguageException("argName " + argName + " must be a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores");
-
- argMap.put("$"+argName,sb.toString());
- }
- else
- {
- // CASE: unnamed argument -- use position in arg list for name
- argMap.put("$"+i ,arg);
- }
- }
-
- return argMap;
- }
-
- /**
- *
- * @param argname
- * @param arg
- * @return
- * @throws IOException
- * @throws LanguageException
- */
- protected static String readDMLScript( String argname, String script )
- throws IOException, LanguageException
- {
- boolean fromFile = argname.equals("-f") ? true : false;
- String dmlScriptStr = null;
-
- if( fromFile )
- {
- //read DML script from file
- if(script == null)
- throw new LanguageException("DML script path was not specified!");
-
- StringBuilder sb = new StringBuilder();
- BufferedReader in = null;
- try
- {
- //read from hdfs or gpfs file system
- if( script.startsWith("hdfs:")
- || script.startsWith("gpfs:") )
- {
- if( !LocalFileUtils.validateExternalFilename(script, true) )
- throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path scriptPath = new Path(script);
- in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
- }
- // from local file system
- else
- {
- if( !LocalFileUtils.validateExternalFilename(script, false) )
- throw new LanguageException("Invalid (non-trustworthy) local filename.");
- in = new BufferedReader(new FileReader(script));
- }
-
- //core script reading
- String tmp = null;
- while ((tmp = in.readLine()) != null)
- {
- sb.append( tmp );
- sb.append( "\n" );
- }
- }
- catch (IOException ex)
- {
- LOG.error("Failed to read the script from the file system", ex);
- throw ex;
- }
- finally
- {
- if( in != null )
- in.close();
- }
-
- dmlScriptStr = sb.toString();
- }
- else
- {
- //parse given script string
- if(script == null)
- throw new LanguageException("DML script was not specified!");
-
- InputStream is = new ByteArrayInputStream(script.getBytes());
- Scanner scan = new Scanner(is);
- dmlScriptStr = scan.useDelimiter("\\A").next();
- scan.close();
- }
-
- return dmlScriptStr;
- }
-
- /**
- *
- * @param platform
- * @return
- */
- private static RUNTIME_PLATFORM parseRuntimePlatform( String platform )
- {
- RUNTIME_PLATFORM lrtplatform = null;
-
- if ( platform.equalsIgnoreCase("hadoop"))
- lrtplatform = RUNTIME_PLATFORM.HADOOP;
- else if ( platform.equalsIgnoreCase("singlenode"))
- lrtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
- else if ( platform.equalsIgnoreCase("hybrid"))
- lrtplatform = RUNTIME_PLATFORM.HYBRID;
- else if ( platform.equalsIgnoreCase("spark"))
- lrtplatform = RUNTIME_PLATFORM.SPARK;
- else if ( platform.equalsIgnoreCase("hybrid_spark"))
- lrtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
- else
- System.err.println("ERROR: Unknown runtime platform: " + platform);
-
- return lrtplatform;
- }
-
- /**
- *
- * @param conf
- */
- private static void setLoggingProperties( Configuration conf )
- {
- String debug = conf.get("systemml.logging");
-
- if (debug == null)
- debug = System.getProperty("systemml.logging");
-
- if (debug != null){
- if (debug.equalsIgnoreCase("debug")){
- Logger.getLogger("com.ibm.bi.dml").setLevel((Level) Level.DEBUG);
- }
- else if (debug.equalsIgnoreCase("trace")){
- Logger.getLogger("com.ibm.bi.dml").setLevel((Level) Level.TRACE);
- }
- }
- }
-
-
- ///////////////////////////////
- // private internal interface
- // (core compilation and execute)
- ////////
-
- /**
- * run: The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
- * and customized parameters have been put into _argVals
- * @throws ParseException
- * @throws IOException
- * @throws DMLRuntimeException
- * @throws HopsException
- * @throws LanguageException
- * @throws DMLUnsupportedOperationException
- * @throws LopsException
- * @throws DMLException
- */
- private static void execute(String dmlScriptStr, String fnameOptConfig, HashMap<String,String> argVals, String[] allArgs, boolean parsePyDML)
- throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException, DMLUnsupportedOperationException
- {
- //print basic time and environment info
- printStartExecInfo( dmlScriptStr );
-
- //Step 1: parse configuration files
- DMLConfig conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(conf);
- LOG.debug("\nDML config: \n" + conf.getConfigInfo());
-
- //Step 2: set local/remote memory if requested (for compile in AM context)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- DMLAppMasterUtils.setupConfigRemoteMaxMemory(conf);
- }
-
- //Step 3: parse dml script
- Statistics.startCompileTimer();
- AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
- DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-
- //Step 4: construct HOP DAGs (incl LVA and validate)
- DMLTranslator dmlt = new DMLTranslator(prog);
- dmlt.liveVariableAnalysis(prog);
- dmlt.validateParseTree(prog);
- dmlt.constructHops(prog);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("\n********************** HOPS DAG (Before Rewrite) *******************");
- dmlt.printHops(prog);
- DMLTranslator.resetHopsDAGVisitStatus(prog);
- }
-
- //Step 5: rewrite HOP DAGs (incl IPA and memory estimates)
- dmlt.rewriteHopsDAG(prog);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("\n********************** HOPS DAG (After Rewrite) *******************");
- dmlt.printHops(prog);
- DMLTranslator.resetHopsDAGVisitStatus(prog);
-
- LOG.debug("\n********************** OPTIMIZER *******************\n" +
- "Level = " + OptimizerUtils.getOptLevel() + "\n"
- +"Available Memory = " + ((double)InfrastructureAnalyzer.getLocalMaxMemory()/1024/1024) + " MB" + "\n"
- +"Memory Budget = " + ((double)OptimizerUtils.getLocalMemBudget()/1024/1024) + " MB" + "\n");
- }
-
- //Step 6: construct lops (incl exec type and op selection)
- dmlt.constructLops(prog);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("\n********************** LOPS DAG *******************");
- dmlt.printLops(prog);
- dmlt.resetLopsDAGVisitStatus(prog);
- }
-
- //Step 7: generate runtime program
- Program rtprog = prog.getRuntimeProgram(conf);
-
- if (LOG.isDebugEnabled()) {
- LOG.info("********************** Instructions *******************");
- rtprog.printMe();
- LOG.info("*******************************************************");
- }
-
- //Step 8: [optional global data flow optimization]
- if(OptimizerUtils.isOptLevel(OptimizationLevel.O4_GLOBAL_TIME_MEMORY) )
- {
- LOG.warn("Optimization level '" + OptimizationLevel.O4_GLOBAL_TIME_MEMORY + "' " +
- "is still in experimental state and not intended for production use.");
- rtprog = GlobalOptimizerWrapper.optimizeProgram(prog, rtprog);
- }
-
- //launch SystemML appmaster (if requested and not already in launched AM)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- if( !isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, conf, allArgs, rtprog) )
- return; //if AM launch unsuccessful, fall back to normal execute
- if( isActiveAM() ) //in AM context (not failed AM launch)
- DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
- }
-
- //Step 9: prepare statistics [and optional explain output]
- //count number compiled MR jobs / SP instructions
- ExplainCounts counts = Explain.countDistributedOperations(rtprog);
- Statistics.resetNoOfCompiledJobs( counts.numJobs );
-
- //explain plan of program (hops or runtime)
- if( EXPLAIN != ExplainType.NONE ) {
- LOG.info("EXPLAIN ("+EXPLAIN.toString()+"):\n"
- + Explain.explainMemoryBudget(counts)+"\n"
- + Explain.explainDegreeOfParallelism(counts)
- + Explain.explain(prog, rtprog, EXPLAIN));
- }
-
- Statistics.stopCompileTimer();
-
- //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
- //System.out.println("Estimated costs: "+costs);
-
-
- //Step 10: execute runtime program
- Statistics.startRunTimer();
- ExecutionContext ec = null;
- try
- {
- initHadoopExecution( conf );
-
- //run execute (w/ exception handling to ensure proper shutdown)
- ec = ExecutionContextFactory.createContext(rtprog);
- rtprog.execute( ec );
-
- }
- finally //ensure cleanup/shutdown
- {
- if(ec != null && ec instanceof SparkExecutionContext) {
- ((SparkExecutionContext) ec).close();
- }
-
- //display statistics (incl caching stats if enabled)
- Statistics.stopRunTimer();
- LOG.info(Statistics.display());
- LOG.info("END DML run " + getDateTime() );
-
- //cleanup scratch_space and all working dirs
- cleanupHadoopExecution( conf );
- }
- }
-
- /**
- * launchDebugger: Launcher for DML debugger. This method should be called after
- * execution and debug properties have been correctly set, and customized parameters
- * have been put into _argVals
- * @param dmlScriptStr DML script contents (including new lines)
- * @param fnameOptConfig Full path of configuration file for SystemML
- * @param argVals Key-value pairs defining arguments of DML script
- * @throws ParseException
- * @throws IOException
- * @throws DMLRuntimeException
- * @throws DMLDebuggerException
- * @throws HopsException
- * @throws LanguageException
- * @throws DMLUnsupportedOperationException
- * @throws LopsException
- * @throws DMLException
- */
- private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, HashMap<String,String> argVals, boolean parsePyDML)
- throws ParseException, IOException, DMLRuntimeException, DMLDebuggerException, LanguageException, HopsException, LopsException, DMLUnsupportedOperationException
- {
- //produce debugging information (parse, compile and generate runtime program for a given DML script)
- DMLDebuggerProgramInfo p = compileForDebug(dmlScriptStr, fnameOptConfig, argVals, parsePyDML);
-
- try {
- //set execution environment
- initHadoopExecution(p.conf);
-
- //initialize an instance of SystemML debugger
- DMLDebugger SystemMLdb = new DMLDebugger(p, dmlScriptStr, argVals);
- //run SystemML debugger
- SystemMLdb.runSystemMLDebugger();
- }
- finally {
- //cleanup scratch_space and all working dirs
- cleanupHadoopExecution(p.conf);
- }
- }
-
- /**
- * compile: Compile DML script and generate hops, lops and runtime program for debugger.
- * This method should be called after execution and debug properties have been set, and
- * customized parameters have been put into _argVals
- * @param dmlScriptStr DML script contents (including new lines)
- * @param fnameOptConfig Full path of configuration file for SystemML
- * @param argVals Key-value pairs defining arguments of DML script
- * @return dbprog Class containing parsed and compiled DML script w/ hops, lops and runtime program
- * @throws ParseException
- * @throws IOException
- * @throws DMLRuntimeException
- * @throws LanguageException
- * @throws HopsException
- * @throws LopsException
- * @throws DMLUnsupportedOperationException
- */
- //TODO: MB: remove this redundant compile and execute (or at least remove from DMLScript)
- //TODO: This method should be private once debugger infrastructure is on top of the programmatic API
- public static DMLDebuggerProgramInfo compileForDebug(String dmlScriptStr, String fnameOptConfig, HashMap<String,String> argVals, boolean parsePyDML)
- throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException, DMLUnsupportedOperationException
- {
- DMLDebuggerProgramInfo dbprog = new DMLDebuggerProgramInfo();
-
- //Step 1: parse configuration files
- dbprog.conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(dbprog.conf);
-
- //Step 2: parse dml script
- AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
- dbprog.prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-
- //Step 3: construct HOP DAGs (incl LVA and validate)
- dbprog.dmlt = new DMLTranslator(dbprog.prog);
- dbprog.dmlt.liveVariableAnalysis(dbprog.prog);
- dbprog.dmlt.validateParseTree(dbprog.prog);
- dbprog.dmlt.constructHops(dbprog.prog);
-
- //Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
- dbprog.dmlt.rewriteHopsDAG(dbprog.prog);
-
- //Step 5: construct LOP DAGs
- dbprog.dmlt.constructLops(dbprog.prog);
-
- //Step 6: generate runtime program
- dbprog.rtprog = dbprog.prog.getRuntimeProgram(dbprog.conf);
-
- return dbprog;
- }
-
- /**
- * @throws ParseException
- * @throws IOException
- * @throws DMLRuntimeException
- *
- */
- static void initHadoopExecution( DMLConfig config )
- throws IOException, ParseException, DMLRuntimeException
- {
- //check security aspects
- checkSecuritySetup( config );
-
- //create scratch space with appropriate permissions
- String scratch = config.getTextValue(DMLConfig.SCRATCH_SPACE);
- MapReduceTool.createDirIfNotExistOnHDFS(scratch, DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);
-
- //cleanup working dirs from previous aborted runs with same pid in order to prevent conflicts
- cleanupHadoopExecution(config);
-
- //init caching (incl set active)
- LocalFileUtils.createWorkingDirectory();
- CacheableData.initCaching();
-
- //reset statistics (required if multiple scripts executed in one JVM)
- Statistics.resetNoOfExecutedJobs( 0 );
- if( STATISTICS ) {
- CacheStatistics.reset();
- Statistics.reset();
- }
- }
-
- /**
- *
- * @param config
- * @throws IOException
- * @throws DMLRuntimeException
- */
- private static void checkSecuritySetup(DMLConfig config)
- throws IOException, DMLRuntimeException
- {
- //analyze local configuration
- String userName = System.getProperty( "user.name" );
- HashSet<String> groupNames = new HashSet<String>();
- try{
- //check existence, for backwards compatibility to < hadoop 0.21
- if( UserGroupInformation.class.getMethod("getCurrentUser") != null ){
- String[] groups = UserGroupInformation.getCurrentUser().getGroupNames();
- for( String g : groups )
- groupNames.add( g );
- }
- }catch(Exception ex){}
-
- //analyze hadoop configuration
- JobConf job = ConfigurationManager.getCachedJobConf();
- boolean localMode = InfrastructureAnalyzer.isLocalMode(job);
- String taskController = job.get("mapred.task.tracker.task-controller", "org.apache.hadoop.mapred.DefaultTaskController");
- String ttGroupName = job.get("mapreduce.tasktracker.group","null");
- String perm = job.get(MRConfigurationNames.DFS_PERMISSIONS,"null"); //note: job.get("dfs.permissions.supergroup",null);
- URI fsURI = FileSystem.getDefaultUri(job);
-
- //determine security states
- boolean flagDiffUser = !( taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user
- || localMode // run in the same JVM anyway
- || groupNames.contains( ttGroupName) ); //user in task tracker group
- boolean flagLocalFS = fsURI==null || fsURI.getScheme().equals("file");
- boolean flagSecurity = perm.equals("yes");
-
- LOG.debug("SystemML security check: " + "local.user.name = " + userName + ", " + "local.user.groups = " + ProgramConverter.serializeStringCollection(groupNames) + ", "
- + "mapred.job.tracker = " + job.get("mapred.job.tracker") + ", " + "mapred.task.tracker.task-controller = " + taskController + "," + "mapreduce.tasktracker.group = " + ttGroupName + ", "
- + "fs.default.name = " + ((fsURI!=null)?fsURI.getScheme():"null") + ", " + MRConfigurationNames.DFS_PERMISSIONS+" = " + perm );
-
- //print warning if permission issues possible
- if( flagDiffUser && ( flagLocalFS || flagSecurity ) )
- {
- LOG.warn("Cannot run map/reduce tasks as user '"+userName+"'. Using tasktracker group '"+ttGroupName+"'.");
- }
-
- //validate external filenames working directories
- String localtmpdir = config.getTextValue(DMLConfig.LOCAL_TMP_DIR);
- String hdfstmpdir = config.getTextValue(DMLConfig.SCRATCH_SPACE);
- if( !LocalFileUtils.validateExternalFilename(localtmpdir, false) )
- throw new DMLRuntimeException("Invalid (non-trustworthy) local working directory.");
- if( !LocalFileUtils.validateExternalFilename(hdfstmpdir, true) )
- throw new DMLRuntimeException("Invalid (non-trustworthy) hdfs working directory.");
- }
-
- /**
- *
- * @param config
- * @throws IOException
- * @throws ParseException
- */
- private static void cleanupHadoopExecution( DMLConfig config )
- throws IOException, ParseException
- {
- //create dml-script-specific suffix
- StringBuilder sb = new StringBuilder();
- sb.append(Lop.FILE_SEPARATOR);
- sb.append(Lop.PROCESS_PREFIX);
- sb.append(DMLScript.getUUID());
- String dirSuffix = sb.toString();
-
- //1) cleanup scratch space (everything for current uuid)
- //(required otherwise export to hdfs would skip assumed unnecessary writes if same name)
- MapReduceTool.deleteFileIfExistOnHDFS( config.getTextValue(DMLConfig.SCRATCH_SPACE) + dirSuffix );
-
- //2) cleanup hadoop working dirs (only required for LocalJobRunner (local job tracker), because
- //this implementation does not create job specific sub directories)
- JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
- if( InfrastructureAnalyzer.isLocalMode(job) ) {
- try
- {
- LocalFileUtils.deleteFileIfExists( DMLConfig.LOCAL_MR_MODE_STAGING_DIR + //staging dir (for local mode only)
- dirSuffix );
- LocalFileUtils.deleteFileIfExists( MRJobConfiguration.getLocalWorkingDirPrefix(job) + //local dir
- dirSuffix );
- MapReduceTool.deleteFileIfExistOnHDFS( MRJobConfiguration.getSystemWorkingDirPrefix(job) + //system dir
- dirSuffix );
- MapReduceTool.deleteFileIfExistOnHDFS( MRJobConfiguration.getStagingWorkingDirPrefix(job) + //staging dir
- dirSuffix );
- }
- catch(Exception ex)
- {
- //we give only a warning because those directories are written by the mapred deamon
- //and hence, execution can still succeed
- LOG.warn("Unable to cleanup hadoop working dirs: "+ex.getMessage());
- }
- }
-
- //3) cleanup systemml-internal working dirs
- CacheableData.cleanupCacheDir(); //might be local/hdfs
- LocalFileUtils.cleanupWorkingDirectory();
- }
-
-
- ///////////////////////////////
- // private internal helper functionalities
- ////////
-
- /**
- *
- * @param fnameScript
- * @param fnameOptConfig
- * @param argVals
- */
- private static void printInvocationInfo(String fnameScript, String fnameOptConfig, HashMap<String,String> argVals)
- {
- LOG.debug("****** args to DML Script ******\n" + "UUID: " + getUUID() + "\n" + "SCRIPT PATH: " + fnameScript + "\n"
- + "RUNTIME: " + rtplatform + "\n" + "BUILTIN CONFIG: " + DMLConfig.DEFAULT_SYSTEMML_CONFIG_FILEPATH + "\n"
- + "OPTIONAL CONFIG: " + fnameOptConfig + "\n");
-
- if( !argVals.isEmpty() ) {
- LOG.debug("Script arguments are: \n");
- for (int i=1; i<= argVals.size(); i++)
- LOG.debug("Script argument $" + i + " = " + argVals.get("$" + i) );
- }
- }
-
- private static void printStartExecInfo(String dmlScriptString)
- {
- LOG.info("BEGIN DML run " + getDateTime());
- LOG.debug("DML script: \n" + dmlScriptString);
-
- if (rtplatform == RUNTIME_PLATFORM.HADOOP || rtplatform == RUNTIME_PLATFORM.HYBRID) {
- String hadoop_home = System.getenv("HADOOP_HOME");
- LOG.info("HADOOP_HOME: " + hadoop_home);
- }
- }
-
- /**
- *
- * @return
- */
- private static String getDateTime()
- {
- DateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
- Date date = new Date();
- return dateFormat.format(date);
- }
-
- /**
- *
- * @throws DMLException
- */
- private static void cleanSystemMLWorkspace()
- throws DMLException
- {
- try
- {
- //read the default config
- DMLConfig conf = DMLConfig.readAndMergeConfigurationFiles(null);
-
- //run cleanup job to clean remote local tmp dirs
- CleanupMR.runJob(conf);
-
- //cleanup scratch space (on HDFS)
- String scratch = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
- if( scratch != null )
- MapReduceTool.deleteFileIfExistOnHDFS(scratch);
-
- //cleanup local working dir
- String localtmp = conf.getTextValue(DMLConfig.LOCAL_TMP_DIR);
- if( localtmp != null )
- LocalFileUtils.cleanupRcWorkingDirectory(localtmp);
- }
- catch(Exception ex)
- {
- throw new DMLException("Failed to run SystemML workspace cleanup.", ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/MLBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/MLBlock.java b/src/main/java/com/ibm/bi/dml/api/MLBlock.java
deleted file mode 100644
index b68f1ad..0000000
--- a/src/main/java/com/ibm/bi/dml/api/MLBlock.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package com.ibm.bi.dml.api;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.StructField;
-import org.apache.spark.sql.types.StructType;
-
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixIndexes;
-
-import scala.collection.Seq;
-
-public class MLBlock implements Row {
-
- private static final long serialVersionUID = -770986277854643424L;
-
- public MatrixIndexes indexes;
- public MatrixBlock block;
-
- public MLBlock(MatrixIndexes indexes, MatrixBlock block) {
- this.indexes = indexes;
- this.block = block;
- }
-
- @Override
- public boolean anyNull() {
- // TODO
- return false;
- }
-
- @Override
- public Object apply(int arg0) {
- if(arg0 == 0) {
- return indexes;
- }
- else if(arg0 == 1) {
- return block;
- }
- // TODO: For now not supporting any operations
- return new Integer(0);
- }
-
- @Override
- public Row copy() {
- return new MLBlock(new MatrixIndexes(indexes), new MatrixBlock(block));
- }
-
- @Override
- public Object get(int arg0) {
- if(arg0 == 0) {
- return indexes;
- }
- else if(arg0 == 1) {
- return block;
- }
- // TODO: For now not supporting any operations
- return new Integer(0);
- }
-
- @Override
- public <T> T getAs(int arg0) {
- // TODO
- return null;
- }
-
- @Override
- public boolean getBoolean(int arg0) {
- // TODO
- return false;
- }
-
- @Override
- public byte getByte(int arg0) {
- // TODO
- return 0;
- }
-
- @Override
- public Date getDate(int arg0) {
- // TODO
- return null;
- }
-
- @Override
- public BigDecimal getDecimal(int arg0) {
- // TODO
- return null;
- }
-
- @Override
- public double getDouble(int arg0) {
- // TODO
- return 0;
- }
-
- @Override
- public float getFloat(int arg0) {
- // TODO
- return 0;
- }
-
- @Override
- public int getInt(int arg0) {
- // TODO
- return 0;
- }
-
- @Override
- public <K, V> Map<K, V> getJavaMap(int arg0) {
- return null;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> List<T> getList(int arg0) {
- ArrayList<Object> retVal = new ArrayList<Object>();
- retVal.add(indexes);
- retVal.add(block);
- //retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(indexes, block));
- return (List<T>) scala.collection.JavaConversions.asScalaBuffer(retVal).toList();
- }
-
- @Override
- public long getLong(int arg0) {
- // TODO
- return 0;
- }
-
- @Override
- public <K, V> scala.collection.Map<K, V> getMap(int arg0) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> Seq<T> getSeq(int arg0) {
- ArrayList<Object> retVal = new ArrayList<Object>();
- retVal.add(indexes);
- retVal.add(block);
- // retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(indexes, block));
- return (Seq<T>) scala.collection.JavaConversions.asScalaBuffer(retVal).toSeq();
- }
-
- @Override
- public short getShort(int arg0) {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public String getString(int arg0) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Row getStruct(int arg0) {
- return this;
- }
-
- @Override
- public boolean isNullAt(int arg0) {
- // TODO Auto-generated method stub
- return false;
- }
-
- @Override
- public int length() {
- return 2;
- }
-
- @Override
- public String mkString() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public String mkString(String arg0) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public String mkString(String arg0, String arg1, String arg2) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public StructType schema() {
- return getDefaultSchemaForBinaryBlock();
- }
-
- @Override
- public int size() {
- return 2;
- }
-
- @Override
- public Seq<Object> toSeq() {
- ArrayList<Object> retVal = new ArrayList<Object>();
- retVal.add(indexes);
- retVal.add(block);
- // retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(indexes, block));
- return scala.collection.JavaConversions.asScalaBuffer(retVal).toSeq();
- }
-
- public static StructType getDefaultSchemaForBinaryBlock() {
- // TODO:
- StructField[] fields = new StructField[2];
- fields[0] = new StructField("IgnoreSchema", DataType.fromCaseClassString("DoubleType"), true, null);
- fields[1] = new StructField("IgnoreSchema1", DataType.fromCaseClassString("DoubleType"), true, null);
- return new StructType(fields);
- }
-
-
-}
[47/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
Posted by lr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/python/SystemML.py b/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
deleted file mode 100644
index a4b22ff..0000000
--- a/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/python
-
-from py4j.protocol import Py4JJavaError, Py4JError
-import traceback
-import os
-from pyspark.sql import *
-from pyspark.rdd import RDD
-
-
-class MLContext(object):
-
- """
- Simple wrapper class for MLContext in SystemML.jar
- ...
- Attributes
- ----------
- ml : MLContext
- A reference to the java MLContext
- sc : SparkContext
- The SparkContext that has been specified during initialization
- """
-
- def __init__(self, sc, *args):
- """
- If initialized with a SparkContext, will connect to the Java MLContext
- class.
- args:
- sc (SparkContext): the current SparkContext
- monitor (boolean=False): Whether to monitor the performance
- forceSpark (boolean=False): Whether to force execution on spark
- returns:
- MLContext: Instance of MLContext
- """
-
- try:
- monitorPerformance = (args[0] if len(args) > 0 else False)
- setForcedSparkExecType = (args[1] if len(args) > 1 else False)
- self.sc = sc
- self.ml = sc._jvm.com.ibm.bi.dml.api.MLContext(sc._jsc, monitorPerformance, setForcedSparkExecType)
- except Py4JError:
- traceback.print_exc()
-
- def reset(self):
- """
- Call this method of you want to clear any RDDs set via
- registerInput or registerOutput
- """
- try:
- self.ml.reset()
- except Py4JJavaError:
- traceback.print_exc()
-
- def execute(self, dmlScriptFilePath, *args):
- """
- Executes the script in spark-mode by passing the arguments to the
- MLContext java class.
- Returns:
- MLOutput: an instance of the MLOutput-class
- """
- numArgs = len(args) + 1
- try:
- if numArgs == 1:
- jmlOut = self.ml.execute(dmlScriptFilePath)
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 2:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 3:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0], args[1])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 4:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0], args[1], args[2])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- else:
- raise TypeError('Arguments do not match MLContext-API')
- except Py4JJavaError:
- traceback.print_exc()
-
- def registerInput(self, varName, src, *args):
- """
- Method to register inputs used by the DML script.
- Supported format:
- 1. DataFrame
- 2. CSV/Text (as JavaRDD<String> or JavaPairRDD<LongWritable, Text>)
- 3. Binary blocked RDD (JavaPairRDD<MatrixIndexes,MatrixBlock>))
- Also overloaded to support metadata information such as format, rlen, clen, ...
- Please note the variable names given below in quotes correspond to the variables in DML script.
- These variables need to have corresponding read/write associated in DML script.
- Currently, only matrix variables are supported through registerInput/registerOutput interface.
- To pass scalar variables, use named/positional arguments (described later) or wrap them into matrix variable.
- """
- numArgs = len(args) + 2
-
- if hasattr(src, '_jdf'):
- rdd = src._jdf
- elif hasattr(src, '_jrdd'):
- rdd = src._jrdd
- else:
- rdd = src
-
- try:
- if numArgs == 2:
- self.ml.registerInput(varName, rdd)
- elif numArgs == 3:
- self.ml.registerInput(varName, rdd, args[0])
- elif numArgs == 4:
- self.ml.registerInput(varName, rdd, args[0], args[1])
- elif numArgs == 5:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2])
- elif numArgs == 6:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3])
- elif numArgs == 7:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3], args[4])
- elif numArgs == 10:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7])
- else:
- raise TypeError('Arguments do not match MLContext-API')
- except Py4JJavaError:
-
- traceback.print_exc()
-
- def registerOutput(self, varName):
- """
- Register output variables used in the DML script
- args:
- varName: (String) The name used in the DML script
- """
-
- try:
- self.ml.registerOutput(varName)
- except Py4JJavaError:
- traceback.print_exc()
-
- def getDmlJson(self):
- try:
- return self.ml.getMonitoringUtil().getRuntimeInfoInJSONFormat()
- except Py4JJavaError:
- traceback.print_exc()
-
-
-class MLOutput(object):
-
- """
- This is a simple wrapper object that returns the output of execute from MLContext
- ...
- Attributes
- ----------
- jmlOut MLContext:
- A reference to the MLOutput object through py4j
- """
-
- def __init__(self, jmlOut, sc):
- self.jmlOut = jmlOut
- self.sc = sc
-
- def getBinaryBlockedRDD(self, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # rdd = RDD(self.jmlOut.getBinaryBlockedRDD(varName), self.sc)
- # return rdd
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getMatrixCharacteristics(self, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # chars = self.jmlOut.getMatrixCharacteristics(varName)
- # return chars
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getDF(self, sqlContext, varName):
- try:
- jdf = self.jmlOut.getDF(sqlContext._scala_SQLContext, varName)
- df = DataFrame(jdf, sqlContext)
- return df
- except Py4JJavaError:
- traceback.print_exc()
-
- def getMLMatrix(self, sqlContext, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # mlm = self.jmlOut.getMLMatrix(sqlContext._scala_SQLContext, varName)
- # return mlm
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getStringRDD(self, varName, format):
- raise Exception('Not supported in Python MLContext')
- #try:
- # rdd = RDD(self.jmlOut.getStringRDD(varName, format), self.sc)
- # return rdd
- #except Py4JJavaError:
- # traceback.print_exc()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java b/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
deleted file mode 100644
index a8fa5f6..0000000
--- a/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.conf;
-
-import org.apache.hadoop.mapred.JobConf;
-
-
-
-/**
- * Singleton for accessing the parsed and merged system configuration.
- *
- * NOTE: parallel execution of multiple DML scripts (in the same JVM) with different configurations
- * would require changes/extensions of this class.
- */
-public class ConfigurationManager
-{
-
- private static DMLConfig _conf = null; //read systemml configuration
- private static JobConf _rJob = null; //cached job conf for read-only operations
-
- static{
- _rJob = new JobConf();
- }
-
-
- /**
- *
- * @param conf
- */
- public synchronized static void setConfig( DMLConfig conf )
- {
- _conf = conf;
- }
-
- /**
- *
- * @return
- */
- public synchronized static DMLConfig getConfig()
- {
- return _conf;
- }
-
- /**
- * Returns a cached JobConf object, intended for global use by all operations
- * with read-only access to job conf. This prevents to read the hadoop conf files
- * over and over again from classpath. However,
- *
- * @return
- */
- public static JobConf getCachedJobConf()
- {
- return _rJob;
- }
-
- /**
- *
- * @param job
- */
- public static void setCachedJobConf(JobConf job)
- {
- _rJob = job;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java b/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
deleted file mode 100644
index 2b54f7b..0000000
--- a/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.conf;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.HashMap;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-
-public class DMLConfig
-{
-
- public static final String DEFAULT_SYSTEMML_CONFIG_FILEPATH = "./SystemML-config.xml";
-
- private static final Log LOG = LogFactory.getLog(DMLConfig.class.getName());
-
- // external names of configuration properties
- // (single point of change for all internal refs)
- public static final String LOCAL_TMP_DIR = "localtmpdir";
- public static final String SCRATCH_SPACE = "scratch";
- public static final String OPTIMIZATION_LEVEL = "optlevel";
- public static final String NUM_REDUCERS = "numreducers";
- public static final String JVM_REUSE = "jvmreuse";
- public static final String DEFAULT_BLOCK_SIZE = "defaultblocksize";
- public static final String YARN_APPMASTER = "dml.yarn.appmaster";
- public static final String YARN_APPMASTERMEM = "dml.yarn.appmaster.mem";
- public static final String YARN_MAPREDUCEMEM = "dml.yarn.mapreduce.mem";
- public static final String YARN_APPQUEUE = "dml.yarn.app.queue";
- public static final String CP_PARALLEL_MATRIXMULT = "cp.parallel.matrixmult";
- public static final String CP_PARALLEL_TEXTIO = "cp.parallel.textio";
-
- //obsolete nimble configuration (removed 06/24/2015)
- //public static final String NUM_MERGE_TASKS = "NumMergeTasks";
- //public static final String NUM_SOW_THREADS = "NumberOfSowThreads";
- //public static final String NUM_REAP_THREADS = "NumberOfReapThreads";
- //public static final String SOWER_WAIT_INTERVAL = "SowerWaitInterval";
- //public static final String REAPER_WAIT_INTERVAL = "ReaperWaitInterval";
- //public static final String NIMBLE_SCRATCH = "NimbleScratch";
-
- //internal config
- public static final String DEFAULT_SHARED_DIR_PERMISSION = "777"; //for local fs and DFS
- public static String LOCAL_MR_MODE_STAGING_DIR = null;
-
- //configuration default values
- private static HashMap<String, String> _defaultVals = null;
-
- private String config_file_name = null;
- private Element xml_root = null;
-
- static
- {
- _defaultVals = new HashMap<String, String>();
- _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
- _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
- _defaultVals.put(OPTIMIZATION_LEVEL, "2" );
- _defaultVals.put(NUM_REDUCERS, "10" );
- _defaultVals.put(JVM_REUSE, "false" );
- _defaultVals.put(DEFAULT_BLOCK_SIZE, "1000" );
- _defaultVals.put(YARN_APPMASTER, "false" );
- _defaultVals.put(YARN_APPMASTERMEM, "2048" );
- _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
- _defaultVals.put(YARN_APPQUEUE, "default" );
- //_defaultVals.put(NUM_MERGE_TASKS, "4" );
- //_defaultVals.put(NUM_SOW_THREADS, "1" );
- //_defaultVals.put(NUM_REAP_THREADS, "1" );
- //_defaultVals.put(SOWER_WAIT_INTERVAL, "1000" );
- //_defaultVals.put(REAPER_WAIT_INTERVAL, "1000" );
- //_defaultVals.put(NIMBLE_SCRATCH, "nimbleoutput" );
- _defaultVals.put(CP_PARALLEL_MATRIXMULT, "true" );
- _defaultVals.put(CP_PARALLEL_TEXTIO, "true" );
- }
-
- public DMLConfig()
- {
-
- }
-
- /**
- *
- * @param fileName
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public DMLConfig(String fileName)
- throws ParseException, FileNotFoundException
- {
- this( fileName, false );
- }
-
- /**
- *
- * @param fileName
- * @param silent
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public DMLConfig(String fileName, boolean silent)
- throws ParseException, FileNotFoundException
- {
- config_file_name = fileName;
- try {
- parseConfig();
- } catch (FileNotFoundException fnfe) {
- LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
- throw fnfe;
- } catch (Exception e){
- //log error, since signature of generated ParseException doesn't allow to pass it
- if( !silent )
- LOG.error("Failed to parse DML config file ",e);
- throw new ParseException("ERROR: error parsing DMLConfig file " + fileName);
- }
-
- LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
- }
-
-
- public String getConfig_file_name()
- {
- return config_file_name;
- }
-
- public DMLConfig( Element root )
- {
- xml_root = root;
- }
-
- public void merge(DMLConfig otherConfig)
- throws ParseException
- {
- if (otherConfig == null)
- return;
-
- try {
- // for each element in otherConfig, either overwrite existing value OR add to defaultConfig
- NodeList otherConfigNodeList = otherConfig.xml_root.getChildNodes();
- if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- for (int i=0; i<otherConfigNodeList.getLength(); i++){
- org.w3c.dom.Node optionalConfigNode = otherConfigNodeList.item(i);
-
- if (optionalConfigNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
-
- // try to find optional config node in default config node
- String paramName = optionalConfigNode.getNodeName();
- String paramValue = ((Element)optionalConfigNode).getFirstChild().getNodeValue();
-
- if (this.xml_root.getElementsByTagName(paramName) != null)
- LOG.info("Updating " + paramName + " with value " + paramValue);
- else
- LOG.info("Defining new attribute" + paramName + " with value " + paramValue);
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
- }
-
- }
- } // end if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- } catch (Exception e){
- LOG.error("Failed in merge default config file with optional config file",e);
- throw new ParseException("ERROR: error merging config file" + otherConfig.config_file_name + " with " + config_file_name);
- }
- }
-
- /**
- * Method to parse configuration
- * @throws ParserConfigurationException
- * @throws SAXException
- * @throws IOException
- */
- private void parseConfig () throws ParserConfigurationException, SAXException, IOException
- {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- factory.setIgnoringComments(true); //ignore XML comments
- DocumentBuilder builder = factory.newDocumentBuilder();
- Document domTree = null;
- if (config_file_name.startsWith("hdfs:") ||
- config_file_name.startsWith("gpfs:") ) // config file from DFS
- {
- if( !LocalFileUtils.validateExternalFilename(config_file_name, true) )
- throw new IOException("Invalid (non-trustworthy) hdfs config filename.");
- FileSystem DFS = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path configFilePath = new Path(config_file_name);
- domTree = builder.parse(DFS.open(configFilePath));
- }
- else // config from local file system
- {
- if( !LocalFileUtils.validateExternalFilename(config_file_name, false) )
- throw new IOException("Invalid (non-trustworthy) local config filename.");
- domTree = builder.parse(config_file_name);
- }
-
- xml_root = domTree.getDocumentElement();
- }
-
- /**
- * Method to get string value of a configuration parameter
- * Handles processing of configuration parameters
- * @param tagName the name of the DMLConfig parameter being retrieved
- * @return a string representation of the DMLConfig parameter value.
- */
- public String getTextValue(String tagName)
- {
- //get the actual value
- String retVal = (xml_root!=null)?getTextValue(xml_root,tagName):null;
-
- if (retVal == null)
- {
- if( _defaultVals.containsKey(tagName) )
- retVal = _defaultVals.get(tagName);
- else
- LOG.error("Error: requested dml configuration property '"+tagName+"' is invalid.");
- }
-
- return retVal;
- }
-
- public int getIntValue( String tagName )
- {
- return Integer.parseInt( getTextValue(tagName) );
- }
-
- public boolean getBooleanValue( String tagName )
- {
- return Boolean.parseBoolean( getTextValue(tagName) );
- }
-
-
- /**
- * Method to get the string value of an element identified by tag
- * @param ele
- * @param tagName
- * @return
- */
- private static String getTextValue(Element element, String tagName) {
- String textVal = null;
- NodeList list = element.getElementsByTagName(tagName);
- if (list != null && list.getLength() > 0) {
- Element elem = (Element) list.item(0);
- textVal = elem.getFirstChild().getNodeValue();
-
- }
- return textVal;
- }
-
- /**
- * Method to update the string value of an element identified by tagname
- * @param ele
- * @param tagName
- * @param newTextValue
- */
- private static void setTextValue(Element element, String tagName, String newTextValue) {
-
- NodeList list = element.getElementsByTagName(tagName);
- if (list != null && list.getLength() > 0) {
- Element elem = (Element) list.item(0);
- elem.getFirstChild().setNodeValue(newTextValue);
- }
- }
-
- /**
- *
- * @return
- * @throws DMLRuntimeException
- */
- public synchronized String serializeDMLConfig()
- throws DMLRuntimeException
- {
- String ret = null;
- try
- {
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
- transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
- //transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- StreamResult result = new StreamResult(new StringWriter());
- DOMSource source = new DOMSource(xml_root);
- transformer.transform(source, result);
- ret = result.getWriter().toString();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to serialize DML config.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @param content
- * @return
- * @throws DMLRuntimeException
- */
- public static DMLConfig parseDMLConfig( String content )
- throws DMLRuntimeException
- {
- DMLConfig ret = null;
- try
- {
- //System.out.println(content);
- DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- Document domTree = null;
- domTree = builder.parse( new ByteArrayInputStream(content.getBytes("utf-8")) );
- Element root = domTree.getDocumentElement();
- ret = new DMLConfig( root );
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to parse DML config.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public static DMLConfig readAndMergeConfigurationFiles( String optConfig )
- throws ParseException, FileNotFoundException
- {
- // optional config specified overwrites/merge into the default config
- DMLConfig defaultConfig = null;
- DMLConfig optionalConfig = null;
-
- if( optConfig != null ) { // the optional config is specified
- try { // try to get the default config first
- defaultConfig = new DMLConfig(DEFAULT_SYSTEMML_CONFIG_FILEPATH, true);
- } catch (FileNotFoundException fnfe) { // it is OK to not have the default, but give a warning
- LOG.warn("No default SystemML config file (" + DEFAULT_SYSTEMML_CONFIG_FILEPATH + ") found");
- } catch (ParseException e) {
- defaultConfig = null;
- throw e;
- }
- try { // try to get the optional config next
- optionalConfig = new DMLConfig(optConfig, false);
- } catch (FileNotFoundException fnfe) {
- LOG.error("Config file " + optConfig + " not found");
- throw fnfe;
- } catch (ParseException e) {
- optionalConfig = null;
- throw e;
- }
- if (defaultConfig != null) {
- try {
- defaultConfig.merge(optionalConfig);
- }
- catch(ParseException e){
- defaultConfig = null;
- throw e;
- }
- }
- else {
- defaultConfig = optionalConfig;
- }
- }
- else { // the optional config is not specified
- try { // try to get the default config
- defaultConfig = new DMLConfig(DEFAULT_SYSTEMML_CONFIG_FILEPATH, false);
- } catch (FileNotFoundException fnfe) { // it is OK to not have the default, but give a warning
- LOG.warn("No default SystemML config file (" + DEFAULT_SYSTEMML_CONFIG_FILEPATH + ") found");
- LOG.warn("Using default settings in DMLConfig");
- DMLConfig dmlConfig = new DMLConfig();
- return dmlConfig;
- } catch (ParseException e) {
- defaultConfig = null;
- throw e;
- }
- }
-
- return defaultConfig;
- }
-
- /**
- *
- * @return
- */
- public String getConfigInfo()
- {
- String[] tmpConfig = new String[] {
- LOCAL_TMP_DIR,SCRATCH_SPACE,OPTIMIZATION_LEVEL,
- NUM_REDUCERS, DEFAULT_BLOCK_SIZE,
- YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM,
- //NUM_MERGE_TASKS, NUM_SOW_THREADS,NUM_REAP_THREADS,
- //SOWER_WAIT_INTERVAL,REAPER_WAIT_INTERVAL,NIMBLE_SCRATCH
- CP_PARALLEL_MATRIXMULT, CP_PARALLEL_TEXTIO
- };
-
- StringBuilder sb = new StringBuilder();
- for( String tmp : tmpConfig )
- {
- sb.append("INFO: ");
- sb.append(tmp);
- sb.append(": ");
- sb.append(getTextValue(tmp));
- sb.append("\n");
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param amMem
- * @param mrMem
- */
- public void updateYarnMemorySettings(String amMem, String mrMem)
- {
- //app master memory
- NodeList list1 = xml_root.getElementsByTagName(YARN_APPMASTERMEM);
- if (list1 != null && list1.getLength() > 0) {
- Element elem = (Element) list1.item(0);
- elem.getFirstChild().setNodeValue(String.valueOf(amMem));
- }
-
- //mapreduce memory
- NodeList list2 = xml_root.getElementsByTagName(YARN_MAPREDUCEMEM);
- if (list2 != null && list2.getLength() > 0) {
- Element elem = (Element) list2.item(0);
- elem.getFirstChild().setNodeValue(String.valueOf(mrMem));
- }
- }
-
- /**
- *
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- public void makeQualifiedScratchSpacePath()
- throws IOException
- {
- NodeList list2 = xml_root.getElementsByTagName(SCRATCH_SPACE);
- if (list2 != null && list2.getLength() > 0) {
- Element elem = (Element) list2.item(0);
-
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- String fname = elem.getFirstChild().getNodeValue();
- Path path = new Path(fname).makeQualified(fs);
-
- elem.getFirstChild().setNodeValue(path.toString());
- }
- }
-
- /**
- *
- * @param key
- * @return
- */
- public static String getDefaultTextValue( String key )
- {
- return _defaultVals.get( key );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java b/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
deleted file mode 100644
index 80b0447..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.TreeMap;
-
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-
-
-/**
- * Class for managing breakpoints within DML compiler and debugger
- */
-public class DMLBreakpointManager {
-
- /** Map between DML script line numbers and breakpoint instructions */
- private static TreeMap<Integer, BreakPointInstruction> breakpoints = new TreeMap<Integer, BreakPointInstruction>();
-
-
- /**
- * Getter for DML breakpoints
- * @return List of breakpoints indexed by DML script line number
- */
- public static TreeMap<Integer, BreakPointInstruction> getBreakpoints()
- {
- if (breakpoints.size() > 0)
- return breakpoints;
- return null;
- }
-
- /**
- * Returns size of active DML breakpoints
- * @return size Size of active breakpoints
- */
- public static int getBreakpointsSize()
- {
- int size = 0;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (breakpoints.get(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE)
- size++;
- }
- return size;
- }
-
- /**
- * Returns breakpoint instruction at a particular line number (if any)
- * @param lineNumber Location of breakpoint
- * @return Breakpoint instruction at indicated line number (if any)
- */
- public static BreakPointInstruction getBreakpoint(int lineNumber) {
- if (!breakpoints.containsKey(lineNumber))
- return null;
- return breakpoints.get(lineNumber);
- }
-
- /**
- * Returns breakpoint instruction with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction at indicated id
- */
- public static BreakPointInstruction getBreakpointAtIndex(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return breakpoints.get(lineNumber);
- }
- }
- return null;
- }
-
- /**
- * Returns breakpoint line number with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction line number (-1 if not found)
- */
- public static int getBreakpointLineNumber(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return lineNumber;
- }
- }
- return -1;
- }
-
- /**
- * Returns breakpoint identifier with given line number
- * @param Line number Location of breakpoint in DML script
- * @return bpID Breakpoint id within all breakpoints (-1 if not found)
- */
- public static int getBreakpointID(int lineNum) {
- int bpID=1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (lineNum == lineNumber) {
- return bpID;
- }
- bpID++;
- }
- return -1;
- }
-
- /**
- * Insert a breakpoint instruction into list of existing breakpoints
- * @param lineNumber Location for inserting breakpoint
- */
- public static void insertBreakpoint (BreakPointInstruction breakpoint, int lineNumber) {
- if (breakpoints.containsKey(lineNumber)) {
- if (breakpoints.get(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE)
- System.out.format("Breakpoint updated at %s, line, %d.\n", breakpoint.getBPInstructionLocation(), lineNumber);
- else
- System.out.format("Breakpoint added at %s, line %d.\n", breakpoint.getBPInstructionLocation(), lineNumber);
- breakpoints.put(lineNumber, breakpoint);
- }
- }
-
- /**
- * Insert a breakpoint instruction into list of breakpoints
- * @param lineNumber Location for inserting breakpoint
- */
- public static void insertBreakpoint (int lineNumber) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(BPINSTRUCTION_STATUS.INVISIBLE);
- }
- else {
- breakpoints.put(lineNumber, new BreakPointInstruction(BPINSTRUCTION_STATUS.INVISIBLE));
- }
- }
-
- /**
- * Updates breakpoint status for a given breakpoint id
- * @param location Breakpoint identifier
- * @param status Current breakpoint status
- */
- public static void updateBreakpoint(int lineNumber, BPINSTRUCTION_STATUS status) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Updates breakpoint status for a given breakpoint id
- * @param location Breakpoint identifier
- * @param status Current breakpoint status
- */
- public static void updateBreakpointID(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Removes breakpoint instruction at given line number
- * @param lineNumber Location for inserting breakpoint
- * @param status Current breakpoint status
- */
- public static void removeBreakpoint(int lineNumber, BPINSTRUCTION_STATUS status) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint deleted at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Removes breakpoint instruction at given location
- * @param location Breakpoint instruction id
- * @param status Current breakpoint status
- */
- public static void removeBreakpointIndex(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1)
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
deleted file mode 100644
index 33d8116..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
+++ /dev/null
@@ -1,609 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.io.PrintStream;
-import java.util.HashMap;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.math.IntRange;
-
-import com.ibm.bi.dml.debug.DMLDebuggerFunctions;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-
-/**
- * This class implements a debugger control module for DML scripts.
- * Note: ONLY USED FOR DEBUGGING PURPOSES
- */
-public class DMLDebugger
-{
-
-
- // This will be supported in subsquent release. turning off the -debug 'optimize' feature for current release.
- //public static boolean ENABLE_DEBUG_OPTIMIZER = false; //default debug mode
- //public static boolean ENABLE_SERVER_SIDE_DEBUG_MODE = false; //default debug mode
-
- //public static final String DEBUGGER_SYSTEMML_CONFIG_FILEPATH = "./DebuggerSystemML-config.xml";
-
-
- private DMLDebuggerProgramInfo dbprog; //parsed and compiled DML script w/ hops, lops and runtime program
- public DMLDebuggerInterface debuggerUI; //debugger command line interface
- private DMLDebuggerFunctions dbFunctions; //debugger functions interface
- private CommandLine cmd; //debugger function command
-
- //support for obtaining STDOUT/STDERR streams of DML program running in debug mode
- private PrintStream originalOut = null;
- private PrintStream originalErr = null;
-
- private String dmlScriptStr; //DML script contents (including new lines)
- HashMap<String,String> argVals; //key-value pairs defining arguments of DML script
- ExecutionContext preEC = null;
- ExecutionContext currEC = null;
- String [] lines;
- volatile boolean quit=false;
-
- /**
- * Constructor for DML debugger CLI
- */
- public DMLDebugger(DMLDebuggerProgramInfo p, String dmlScript, HashMap<String,String> args)
- {
- dbprog = p;
- dmlScriptStr = dmlScript;
- lines = dmlScriptStr.split("\n");
- argVals = args;
- debuggerUI = new DMLDebuggerInterface();
- dbFunctions = new DMLDebuggerFunctions();
- preEC = ExecutionContextFactory.createContext(dbprog.rtprog);
- setupDMLRuntime();
- }
-
- /**
- * Sets up DML runtime with DML script and instructions information
- */
- private void setupDMLRuntime()
- {
- dbprog.setDMLInstMap();
- preEC.getDebugState().setDMLScript(lines);
- }
-
- /**
- * Sets STDOUT stream of a DML program running in debug mode
- */
- @SuppressWarnings("unused")
- private void setStdOut()
- {
- originalOut = System.out;
- System.setOut(originalOut);
- }
-
- /**
- * Gets STDOUT stream of a DML program running in debug mode
- * @return STDOUT stream of DML program
- */
- @SuppressWarnings("unused")
- private PrintStream getStdOut()
- {
- System.out.flush();
- return originalOut;
- }
-
- /**
- * Sets STDERR stream of a DML program running in debug mode
- */
- @SuppressWarnings("unused")
- private void setStdErr()
- {
- originalErr = System.err;
- System.setOut(originalErr);
- }
-
- /**
- * Gets STDERR stream of a DML program running in debug mode
- * @return STDERR stream of DML program
- */
- @SuppressWarnings("unused")
- private PrintStream getStdErr()
- {
- System.err.flush();
- return originalErr;
- }
-
- /**
- * Get debug function command from debugger CLI
- * @throws DMLDebuggerException
- */
- private void getCommand() throws DMLDebuggerException
- {
- cmd = debuggerUI.getDebuggerCommand();
- }
-
- /**
- * Class for running the DML runtime as a thread
- */
- Runnable DMLRuntime = new Runnable()
- {
- public void run() {
- try {
- // System.out.println("Starting DML script ...");
- dbprog.rtprog.execute(currEC);
- // System.out.println("DML script has finished execution.");
- synchronized(DMLDebugger.class) {
- quit = true;
- }
- }
- catch (Exception e) {
- System.err.println("Exception raised by DML runtime:" + e);
- }
- }
- };
-
- /**
- * Controls the communication between debugger CLI and DML runtime.
- */
- @SuppressWarnings("deprecation")
- public synchronized void runSystemMLDebugger()
- {
- debuggerUI.setOptions();
- debuggerUI.getDebuggerCLI();
- Thread runtime = new Thread(DMLRuntime);
- boolean isRuntimeInstruction = false;
-
- while (!quit) {
- try {
- //get debugger function from CLI
- getCommand();
- if(cmd != null) {
- isRuntimeInstruction = false;
- //check for help
- if(cmd.hasOption("h")) {
- debuggerUI.getDebuggerCLI();
- }
- //check for exit
- else if (cmd.hasOption("q")) {
- synchronized(DMLDebugger.class) {
- quit = true;
- }
- runtime.stop();
- }
- else if (cmd.hasOption("r")) {
- if (currEC != null) {
- System.out.println("Runtime has already started. Try \"s\" to go to next line, or \"c\" to continue running your DML script.");
- }
- else {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- }
- }
- else if (cmd.hasOption("c")) {
- if (currEC == null)
- System.out.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else if (!runtime.isAlive()) {
- System.err.println("Invalid debug state.");
- //System.out.println("Runtime terminated. Try \"-c\" to recompile followed by \"r\" to restart DML runtime execution.");
- }
- else {
- System.out.println("Resuming DML script execution ...");
- preEC.getDebugState().setCommand(null);
- runtime.resume();
- isRuntimeInstruction = true;
- }
- }
- else if (cmd.hasOption("si")) {
- if (!runtime.isAlive()) {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- // System.out.println("Runtime must be started before single stepping can be enabled. Try \"r\" to start DML runtime execution.");
- }
- //else {
- preEC.getDebugState().setCommand("step_instruction");
- runtime.resume();
- isRuntimeInstruction = true;
- //}
- }
- else if (cmd.hasOption("s")) {
- if (!runtime.isAlive()) {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- //System.out.println("Runtime must be started before step over can be enabled. Try \"r\" to start DML runtime execution.");
- }
- //else {
- preEC.getDebugState().setCommand("step_line");
- runtime.resume();
- isRuntimeInstruction = true;
- //}
- }
-// else if (cmd.hasOption("step_return")) {
-// if (!runtime.isAlive()) {
-// System.out.println("Runtime must be started before step return can be enabled. Try \"r\" to start DML runtime execution.");
-// }
-// else {
-// String fname = dbFunctions.getValue(cmd.getOptionValues("step_return"));
-// dbprog.rtprog.setCommand("step return");
-// if (fname != null) {
-// dbprog.rtprog.setCommandArg(fname);
-// }
-// runtime.resume();
-// isRuntimeInstruction = true;
-// }
-// }
- else if (cmd.hasOption("b")) {
- int lineNumber = dbFunctions.getValue(cmd.getOptionValues("b"), lines.length);
- if (lineNumber > 0) {
- if (DMLBreakpointManager.getBreakpoint(lineNumber) == null)
- System.out.println("Sorry, a breakpoint cannot be inserted at line " + lineNumber + ". Please try a different line number.");
- else {
- if (DMLBreakpointManager.getBreakpoint(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) {
- System.out.format("Breakpoint at line %d already exists.\n", lineNumber);
- }
- else {
- dbprog.accessBreakpoint(lineNumber, 0, BPINSTRUCTION_STATUS.ENABLED);
- }
- }
- }
- }
- else if (cmd.hasOption("d")) {
- int lineNumber = dbFunctions.getValue(cmd.getOptionValues("d"), lines.length);
- if (lineNumber > 0 && DMLBreakpointManager.getBreakpoint(lineNumber) != null &&
- DMLBreakpointManager.getBreakpoint(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) {
- dbprog.accessBreakpoint(lineNumber, 1, BPINSTRUCTION_STATUS.INVISIBLE);
- //dbprog.accessBreakpoint(lineNumber, 1, BPINSTRUCTION_STATUS.DISABLED);
- }
- else {
- System.out.println("Sorry, a breakpoint cannot be deleted at line " + lineNumber + ". Please try a different line number.");
- }
-
- }
- else if (cmd.hasOption("i")) {
- String [] infoOptions = cmd.getOptionValues("i");
- if(infoOptions == null || infoOptions.length == 0) {
- System.err.println("The command \"info\" requires option. Try \"info break\" or \"info frame\".");
- }
- else if(infoOptions[0].trim().compareTo("break") == 0) {
- dbFunctions.listBreakpoints(DMLBreakpointManager.getBreakpoints());
- }
- else if(infoOptions[0].trim().compareTo("frame") == 0) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution.");
- else
- dbFunctions.printCallStack(currEC.getDebugState().getCurrentFrame(), currEC.getDebugState().getCallStack());
- }
- else {
- System.err.println("Invalid option for command \"info\". Try \"info break\" or \"info frame\".");
- }
- }
- else if (cmd.hasOption("p")) {
- String [] pOptions = cmd.getOptionValues("p");
- if(pOptions == null || pOptions.length != 1) {
- System.err.println("Incorrect options for command \"print\"");
- }
- else {
- String varName = pOptions[0].trim();
- if (runtime.isAlive()) {
- if(varName.contains("[")) {
- // matrix with index: can be cell or column or row
- try {
- //System.out.println("" + varName);
- String variableNameWithoutIndices = varName.split("\\[")[0].trim();
- //System.out.println("" + variableNameWithoutIndices);
- String indexString = (varName.split("\\[")[1].trim()).split("\\]")[0].trim();
- //System.out.println(">>" + indexString + "<<");
- String rowIndexStr = "";
- String colIndexStr = "";
- if(indexString.startsWith(",")) {
-
- colIndexStr = indexString.split(",")[1].trim();
- }
- else if(indexString.endsWith(",")) {
- rowIndexStr = indexString.split(",")[0].trim();
- }
- else {
- rowIndexStr = indexString.split(",")[0].trim();
- colIndexStr = indexString.split(",")[1].trim();
- }
- int rowIndex = -1;
- int colIndex = -1;
- if(rowIndexStr.compareTo("") != 0) {
- rowIndex = Integer.parseInt(rowIndexStr);
- }
- if(colIndexStr.compareTo("") != 0) {
- colIndex = Integer.parseInt(colIndexStr);
- }
- //System.out.println("" + rowIndex + " " + colIndex);
- dbFunctions.print(currEC.getDebugState().getVariables(), variableNameWithoutIndices, "value", rowIndex, colIndex);
- }
- catch(Exception indicesException) {
- System.err.println("Incorrect fomat for \"p\". If you are trying to print matrix variable M, you can use M[1,] or M[,1] or M[1,1] (without spaces).");
- }
- }
- else {
- // Print entire matrix
- dbFunctions.print(currEC.getDebugState().getVariables(), varName, "value", -1, -1);
- }
- }
- else
- System.err.println("Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution.");
- }
- }
- else if (cmd.hasOption("whatis")) {
- String [] pOptions = cmd.getOptionValues("whatis");
- if(pOptions == null || pOptions.length != 1) {
- System.err.println("Incorrect options for command \"whatis\"");
- }
- else {
- String varName = pOptions[0].trim();
- dbFunctions.print(currEC.getDebugState().getVariables(), varName, "metadata", -1, -1);
- }
- }
- else if (cmd.hasOption("set")) {
- String [] pOptions = cmd.getOptionValues("set");
- if(pOptions == null || pOptions.length != 2) {
- System.err.println("Incorrect options for command \"set\"");
- }
- else {
- try {
- if(pOptions[0].contains("[")) {
- String [] paramsToSetMatrix = new String[4];
- paramsToSetMatrix[0] = pOptions[0].split("\\[")[0].trim();
- String indexString = (pOptions[0].split("\\[")[1].trim()).split("\\]")[0].trim();
- paramsToSetMatrix[1] = indexString.split(",")[0].trim();
- paramsToSetMatrix[2] = indexString.split(",")[1].trim();
- paramsToSetMatrix[3] = pOptions[1].trim();
- dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), paramsToSetMatrix);
- }
- else {
- dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), pOptions);
- }
- }
- catch(Exception exception1) {
- System.out.println("Only scalar variable or a matrix cell available in current frame can be set in current version.");
- }
- }
- }
- else if (cmd.hasOption("l")) {
-
- String [] pOptions = cmd.getOptionValues("l");
- String [] argsForRange = new String[2];
- int currentPC = 1;
-
- if(runtime.isAlive()) {
- currentPC = currEC.getDebugState().getPC().getLineNumber();
- }
-
- IntRange range = null;
- if(pOptions == null) {
- // Print first 10 lines
- range = new IntRange(currentPC, Math.min(lines.length, currentPC+10));
- }
- else if(pOptions.length == 1 && pOptions[0].trim().toLowerCase().compareTo("all") == 0) {
- // Print entire program
- range = new IntRange(1, lines.length);
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("next") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + currentPC;
- argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC);
- range = dbFunctions.getRange(argsForRange, lines.length);
-
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("prev") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + Math.max(1, currentPC - numLines);
- argsForRange[1] = "" + currentPC;
- range = dbFunctions.getRange(argsForRange, lines.length);
- }
-
-
- if(range == null) {
- System.err.println("Incorrect usage of command \"l\". Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\".");
- }
- else {
- if (range.getMinimumInteger() > 0) {
- dbFunctions.printLines(lines, range);
- }
- else {
- System.err.println("Sorry no lines that can be printed. Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\".");
- }
- }
-
-
-
- // Old code:
- // IntRange range = dbFunctions.getRange(cmd.getOptionValues("p"), lines.length);
- //if (range.getMinimumInteger() > 0) {
- // dbFunctions.printLines(lines, range);
- // }
- }
- else if (cmd.hasOption("li")) {
-
- String [] pOptions = cmd.getOptionValues("li");
- String [] argsForRange = new String[2];
- int currentPC = 1;
-
- if(runtime.isAlive()) {
- currentPC = currEC.getDebugState().getPC().getLineNumber();
- }
-
- IntRange range = null;
- if(pOptions == null) {
- // Print first 10 lines
- range = new IntRange(currentPC, Math.min(lines.length, currentPC+10));
- }
- else if(pOptions.length == 1 && pOptions[0].trim().toLowerCase().compareTo("all") == 0) {
- // Print entire program
- range = new IntRange(1, lines.length);
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("next") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + currentPC;
- argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC);
- range = dbFunctions.getRange(argsForRange, lines.length);
-
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("prev") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + Math.max(1, currentPC - numLines);
- argsForRange[1] = "" + currentPC;
- range = dbFunctions.getRange(argsForRange, lines.length);
- }
-
-
- if(range == null) {
- System.err.println("Incorrect usage of command \"li\". Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\".");
- }
- else {
- if (range.getMinimumInteger() > 0) {
- dbFunctions.printInstructions(lines, dbprog.getDMLInstMap(), range, false);
- }
- else {
- System.err.println("Sorry no lines that can be printed. Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\".");
- }
- }
-
-
-
- // Old code:
- // IntRange range = dbFunctions.getRange(cmd.getOptionValues("p"), lines.length);
- //if (range.getMinimumInteger() > 0) {
- // dbFunctions.printLines(lines, range);
- // }
- }
- else if (cmd.hasOption("set_scalar")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else
- dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_scalar"));
- }
- else if (cmd.hasOption("m")) {
- String varname = dbFunctions.getValue(cmd.getOptionValues("m"));
- if (runtime.isAlive())
- dbFunctions.printMatrixVariable(currEC.getDebugState().getVariables(), varname);
- else
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- }
- else if (cmd.hasOption("x")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else {
- dbFunctions.printMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("x"));
- }
- }
- else if (cmd.hasOption("set_cell")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else {
- dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_cell"));
- }
- }
- else {
- System.err.println("Undefined command. Try \"help\".");
- }
- //block until runtime suspends execution or terminates
- //while(runtime.isAlive() && !currEC.getProgram().isStopped()) {
- wait(300); // To avoid race condition between submitting job and
- //System.out.println(">> Before while");
- while(isRuntimeInstruction && !currEC.getDebugState().canAcceptNextCommand()) {
- if(quit){
- break;
- }
- else {
- wait(300); //wait
- }
- }
- //System.out.println(">> After while");
- }
- wait(300);
- } catch (Exception e) {
- System.err.println("Error processing debugger command. Try \"help\".");
- }
- }
- }
-
-
-// Since the recompile option is disabled in the debugger to make the interface much cleaner
-// /**
-// * compile: Compile DML script and generate hops, lops and runtime program for debugger.
-// * @param dmlScriptStr DML script contents (including new lines)
-// * @param argVals Key-value pairs defining arguments of DML script
-// * @throws ParseException
-// * @throws IOException
-// * @throws DMLRuntimeException
-// * @throws LanguageException
-// * @throws HopsException
-// * @throws LopsException
-// * @throws DMLUnsupportedOperationException
-// * @throws Exception
-// */
-// private void recompile(String dmlScriptStr, HashMap<String,String> argVals)
-// throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException, DMLUnsupportedOperationException, Exception
-// {
-// dbprog = new DMLDebuggerProgramInfo();
-//
-// //Step 1: parse dml script
-// if(DMLScript.ENABLE_PYTHON_PARSING) {
-// PyDMLParserWrapper parser = new PyDMLParserWrapper();
-// dbprog.prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-// }
-// else {
-// DMLParserWrapper parser = new DMLParserWrapper();
-// dbprog.prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-// }
-//
-// //Step 3: construct HOP DAGs (incl LVA and validate)
-// dbprog.dmlt = new DMLTranslator(dbprog.prog);
-// dbprog.dmlt.liveVariableAnalysis(dbprog.prog);
-// dbprog.dmlt.validateParseTree(dbprog.prog);
-// dbprog.dmlt.constructHops(dbprog.prog);
-//
-// //Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
-// dbprog.dmlt.rewriteHopsDAG(dbprog.prog);
-//
-// //Step 5: construct LOP DAGs
-// dbprog.dmlt.constructLops(dbprog.prog);
-//
-// //Step 6: generate runtime program
-// dbprog.rtprog = dbprog.prog.getRuntimeProgram(dbprog.conf);
-//
-// //Set debug mode flag
-// setupDMLRuntime();
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
deleted file mode 100644
index 132a5c7..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * This exception should be thrown to flag debugger errors.
- */
-public class DMLDebuggerException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLDebuggerException(String string) {
- super(string);
- }
-
- public DMLDebuggerException(Exception e) {
- super(e);
- }
-
- public DMLDebuggerException(String string, Exception ex){
- super(string,ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
deleted file mode 100644
index 94ce010..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
+++ /dev/null
@@ -1,678 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.ArrayList;
-import java.util.Map.Entry;
-import java.util.Stack;
-import java.util.TreeMap;
-
-import org.apache.commons.lang.math.IntRange;
-
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-
-public class DMLDebuggerFunctions {
-
- //Maximum values of rows and columns when displaying a DML matrix/vector
- public static final int DISPLAY_MAX_ROWS = 30;
- public static final int DISPLAY_MAX_COLUMNS = 8;
-
- /////////////////////////////////////////////
- // public interface for debugger functions //
- /////////////////////////////////////////////
-
- /**
- * Print all breakpoints along with current status (i.e. enabled or disabled)
- * @param breakpoints Contains all existing breakpoints
- */
- public void listBreakpoints(TreeMap<Integer, BreakPointInstruction> breakpoints)
- {
- //Display all breakpoints
- if (breakpoints == null)
- {
- System.out.println("No breakpoints are set for this program.");
- return;
- }
- int currBreakpoint = 1; //active breakpoint ids
- int numVisibleBreakpoints = 0;
- for (Entry<Integer, BreakPointInstruction> e : breakpoints.entrySet() )
- {
- Integer lineNumber = e.getKey();
- BreakPointInstruction inst = e.getValue();
-
- if (inst.getBPInstructionStatus() == BPINSTRUCTION_STATUS.ENABLED) {
- System.out.format("Breakpoint %2d, at line %4d (%s)\n", currBreakpoint++, lineNumber, "enabled");
- numVisibleBreakpoints++;
- }
- else if (inst.getBPInstructionStatus() == BPINSTRUCTION_STATUS.DISABLED) {
- System.out.format("Breakpoint %2d, at line %4d (%s)\n", currBreakpoint++, lineNumber, "disabled");
- numVisibleBreakpoints++;
- }
- }
-
- if(numVisibleBreakpoints == 0) {
- System.out.println("No breakpoints are set for this program.");
- }
- }
-
- /**
- * Print range of DML program lines
- * @param lines DML script lines of code
- * @param range Range of lines of DML code to be displayed
- */
- public void printLines(String [] lines, IntRange range)
- {
- //Display all lines of DML script
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- System.out.format("line %4d: %s\n", lineNumber, lines[lineNumber-1]);
- }
-
- /**
- * Print range of DML program lines interspersed with corresponding runtime instructions
- * @param lines DML script lines of code
- * @param DMLInstMap Mapping between source code line number and corresponding runtime instruction(s)
- * @param range Range of lines of DML code to be displayed
- * @param debug Flag for displaying instructions in debugger test integration
- */
- public void printInstructions(String [] lines, TreeMap<Integer, ArrayList<Instruction>> DMLInstMap, IntRange range, boolean debug)
- {
- //Display instructions with corresponding DML line numbers
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- {
- System.out.format("line %4d: %s\n", lineNumber, lines[lineNumber-1]);
- if (DMLInstMap.get(lineNumber) != null)
- {
- for (Instruction currInst : DMLInstMap.get(lineNumber))
- {
- if (currInst instanceof CPInstruction) {
- if (!debug)
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currInst.toString()));
- else {
- String [] instStr = prepareInstruction(currInst.toString()).split(" ");
- System.out.format("\t\t id %4d: %s %s\n", currInst.getInstID(), instStr[0], instStr[1]);
- }
- }
- else if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currMRInst.getMRString(debug)));
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), currBPInst.toString());
- }
- }
- }
- }
- }
-
- /**
- * Print range of program runtime instructions
- * @param DMLInstMap Mapping between source code line number and corresponding runtime instruction(s)
- * @param range Range of lines of DML code to be displayed
- */
- public void printRuntimeInstructions(TreeMap<Integer, ArrayList<Instruction>> DMLInstMap, IntRange range)
- {
- //Display instructions
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- {
- if (DMLInstMap.get(lineNumber) != null)
- {
- for (Instruction currInst : DMLInstMap.get(lineNumber))
- {
- if (currInst instanceof CPInstruction)
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currInst.toString()));
- else if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currMRInst.getMRString(false)));
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), currBPInst.toString());
- }
- }
- }
- }
- }
-
- /**
- * Print current program counter
- * @param pc Current stack frame program counter
- */
- public void printPC(DMLProgramCounter pc)
- {
- //Display program counter
- if (pc != null)
- System.out.println(" Current program counter at " + pc.toString());
- else
- System.out.println("DML runtime is currently inactive.");
- }
-
- /**
- * Print current stack frame variables
- * @param frameVariables Current frame variables
- */
- public void printFrameVariables(LocalVariableMap frameVariables)
- {
- //Display local stack frame variables
- if (frameVariables != null && !frameVariables.keySet().isEmpty()) {
- System.out.println(" Local variables:");
- System.out.format("\t%-40s %-40s", "Variable name", "Variable value" );
- for( String varname : frameVariables.keySet())
- System.out.format("\n\t%-40s %-40s", varname, frameVariables.get(varname).toString());
- System.out.println();
- }
- else
- System.out.println("\tSymbol table for current frame is empty");
- }
-
- /**
- * Print current stack frame information
- * @param frame Current stack frame which contains pc and local variables
- */
- public void printFrame(DMLFrame frame) {
- if (frame != null && frame.getPC() != null) {
- printPC(frame.getPC());
- printFrameVariables(frame.getVariables());
- }
- else
- System.out.println("DML runtime is currently inactive.");
- }
-
- /**
- * Print current call stack
- * @param currentEC Current stack frame
- * @param callStack Saved stack frames
- */
- public void printCallStack(DMLFrame currFrame, Stack<DMLFrame> callStack) {
- int frameID = 0;
- if (currFrame == null)
- System.out.println("DML runtime is currently inactive.");
- else {
- if (callStack != null) {
- for(DMLFrame frame : callStack) {
- System.out.println("Frame id: " + frameID++);
- printFrame(frame);
- }
- }
- System.out.println("Current frame id: " + frameID++);
- printFrame(currFrame);
- }
- }
-
- /**
- * Print DML scalar variable in current frame (if existing)
- * @param variables Current frame variables
- * @param varname Variable name
- */
- public void printScalarVariable(LocalVariableMap variables, String varname) {
- if (varname == null) {
- System.err.println("No scalar variable name entered.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.SCALAR)
- System.out.println(varname + " = " + variables.get(varname).toString());
- else
- System.out.println("Variable \""+varname+"\" is not scalar variable.");
- }
- else
- System.out.println("DML scalar variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Set value of specified DML scalar variable in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and value)
- */
- public void setScalarValue(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.SCALAR) {
- Data value;
- //try {
- switch(variables.get(varname).getValueType()) {
- case DOUBLE:
- double d = Double.parseDouble(args[1]);
- value = (ScalarObject) new DoubleObject(d);
- break;
- case INT:
- long i = Long.parseLong(args[1]);
- value = (ScalarObject) new IntObject(i);
- break;
- case BOOLEAN:
- boolean b = Boolean.parseBoolean(args[1]);
- value = (ScalarObject) new BooleanObject(b);
- break;
- case STRING:
- value = (ScalarObject) new StringObject(args[1]);
- break;
- default:
- System.err.println("Invalid scalar value type.");
- return;
- }
- //}
-// catch (Exception e) {
-// System.err.println("Error processing set scalar value command for variable"+varname+".");
-// return;
-// }
- variables.put(varname, value);
- System.out.println(varname + " = " + variables.get(varname).toString());
- }
- else
- System.out.println("Variable \""+varname+"\" is not scalar variable.");
- }
- else
- System.out.println("DML scalar variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- public void print(LocalVariableMap variables, String varname, String displayFunction, int rowIndex, int colIndex) throws DMLRuntimeException {
- if (varname == null) {
- System.err.println("No matrix variable name entered.");
- return;
- }
-
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- try {
- MatrixObject mo = null;
-
- try {
- mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: Matrix dimensions are too large to fit in main memory.");
- return;
- }
- }
- catch(Exception fetchMatrixException) {
- System.err.println("ERROR: While fetching the matrix from symbol table.");
- return;
- }
-
- if(displayFunction.compareTo("value") == 0) {
- MatrixBlock mb = null;
- try {
- // Only read the MatrixBlock when asked to print, but not for whatis
- mb = mo.acquireRead();
- prettyPrintMatrixBlock(mb, rowIndex, colIndex);
- mo.release();
- }
- catch(Exception fetchMatrixException) {
- System.err.println("ERROR: Matrix dimensions are too large to fit in main memory.");
- return;
- }
- }
- else if(displayFunction.compareTo("metadata") == 0) {
- System.out.println("Metadata of " + varname + ": matrix"+variables.get(varname).getMetaData().toString());
- }
-
-
- } catch (Exception e) {
- String command = "";
- if(displayFunction.compareTo("value") == 0) {
- command = "print";
- }
- else {
- command = "whatis";
- }
- System.err.println("Error processing \'" + command + "\' command for variable "+varname+".");
- return;
- }
- }
- else if (variables.get(varname).getDataType() == DataType.SCALAR) {
- if(displayFunction.compareTo("value") == 0) {
- System.out.println(varname + " = " + variables.get(varname).toString());
- }
- else if(displayFunction.compareTo("metadata") == 0) {
- System.out.println("Metadata of " + varname + ": DataType.SCALAR");
- }
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector or scalar variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"info frame\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
-
-
- /**
- * Print DML matrix variable in current frame (if existing)
- * @param variables Current frame variables
- * @param varname Variable name
- * @throws DMLRuntimeException
- */
- public void printMatrixVariable(LocalVariableMap variables, String varname) throws DMLRuntimeException {
- if (varname == null) {
- System.err.println("No matrix variable name entered.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireRead();
- prettyPrintMatrixBlock(mb, -1, -1);
- mo.release();
- if (mb.getNumRows() > DISPLAY_MAX_ROWS || mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) and %d column(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS), min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- System.out.println("Metadata: "+variables.get(varname).getMetaData().toString());
- } catch (Exception e) {
- System.err.println("Error processing display DML matrix command for variable "+varname+".");
- return;
- }
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Print DML matrix cell contents in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and row and column indexes)
- */
- public void printMatrixCell(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- int rowIndex, columnIndex;
- try {
- rowIndex = Integer.parseInt(args[1]);
- columnIndex = Integer.parseInt(args[2]);
- } catch (Exception e) {
- System.err.print("Invalid display cell arguments.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- double cellValue;
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireRead();
- cellValue = mb.getValue(rowIndex, columnIndex);
- mo.release();
- } catch (Exception e) {
- System.err.println("Error processing DML matrix variable "+varname+". Certain matrix operations are disabled due to memory constraints or read-only restrictions.");
- return;
- }
- System.out.println(varname+"["+rowIndex+","+columnIndex+"] = "+cellValue);
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Set DML matrix cell contents in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and row and column indexes)
- */
- public void setMatrixCell(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- int rowIndex, columnIndex;
- double value;
- try {
- rowIndex = Integer.parseInt(args[1]) - 1;
- columnIndex = Integer.parseInt(args[2]) - 1;
- value = Double.parseDouble(args[3]);
- } catch (Exception e) {
- System.err.print("Invalid set cell arguments.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- double updatedCellValue;
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireModify();
- mb.setValue(rowIndex, columnIndex, value);
- updatedCellValue = mb.getValue(rowIndex, columnIndex);
- mo.release();
- } catch (Exception e) {
- System.err.println("Error processing DML matrix variable "+varname+". Certain matrix operations are disabled due to memory constraints or read-only restrictions.");
- return;
- }
- System.out.println(varname+"["+ (rowIndex+1) +","+ (columnIndex+1) +"] = "+updatedCellValue);
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /////////////////////////////////////////
- // internal debugger parsing functions //
- /////////////////////////////////////////
-
- /**
- * Parse command line argument and return valid value
- * @param args CLI arguments for current debugger function
- * @return Validated value for current debug command
- */
- protected String getValue(String[] args) {
- String value = null;
- if (args != null) {
- if (args.length > 1)
- System.err.println("Invalid number of argument values for this command. Try \"help\".");
- value = args[0];
- }
- return value;
- }
-
- /**
- * Parse command line argument and return valid integer
- * @param args CLI arguments for current debugger function
- * @param length Maximum value for input parameter
- * @return Validated integer value for current debug command
- */
- protected int getValue(String[] args, int length) {
- int lineNum = 0;
- if (args == null || args.length > 1) {
- System.err.print("Invalid argument value for this command. Parameter must be a positive integer <= "+length);
- }
- else {
- try {
- lineNum = Integer.parseInt(args[0]);
- if (lineNum <= 0 || lineNum > length)
- {
- System.err.println("Invalid argument value for this command. Parameter must be a positive integer <= "+length);
- lineNum = 0;
- }
- } catch (NumberFormatException e) {
- System.err.println("Invalid integer format. Parameter must be a positive integer <= "+length);
- lineNum = 0;
- }
- }
- return lineNum;
- }
-
- /**
- * Parse command line arguments and return valid IntRange variable
- * @param args CLI arguments for range of debugger display functionality
- * @param size Size (number of lines of code) of DML script
- * @return Validated range of lines within DML script to be displayed
- * @throws DMLDebuggerException Invalid range
- */
- protected IntRange getRange(String [] args, int length) throws DMLDebuggerException {
- IntRange range = new IntRange(1, length);
- if (args == null)
- return range;
- if (args.length == 2) {
- try {
- range = new IntRange(Integer.parseInt(args[0]), Integer.parseInt(args[1]));
- if (range.getMinimumInteger() <= 0 || range.getMaximumInteger() > length) {
- System.err.println("Invalid range values. Parameters <start end> must be two positive integers.");
- range = new IntRange(0, 0);
- }
- } catch (NumberFormatException e) {
- System.err.println("Invalid integer range format. Parameter must be a positive integer <= "+length);
- range = new IntRange(0, 0);
- }
- }
- else {
- System.err.println("Invalid range values. Parameters <start end> must be two positive integers.");
- range = new IntRange(0, 0);
- }
- return range;
- }
-
- /**
- * Returns minimum between two integers
- * @param a Integer value
- * @param b Integer value
- * @return Minimum between a and b.
- */
- private int min(int a, int b) {
- if (a < b)
- return a;
- return b;
- }
-
- /**
- * Displays a pretty-printed version of a DML matrix or vector variable.
- * @param mb Current matrix block
- * @param rowIndex if rowIndex == -1, then prints all rows
- * @param colIndex if colIndex == -1, then prints all columns
- */
- private void prettyPrintMatrixBlock(MatrixBlock mb, int rowIndex, int colIndex) {
- if(rowIndex <= 0 && colIndex <= 0) {
- // Print entire matrix
- for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- for(int j=0; j<min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS); j++) {
- System.out.format("%.4f\t", mb.quickGetValue(i, j));
- }
- System.out.println();
- }
- if (mb.getNumRows() > DISPLAY_MAX_ROWS || mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) and %d column(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS), min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- }
- else if(rowIndex >= 0 && colIndex >= 0) {
- // Print a cell
- System.out.format("%.4f\n", mb.quickGetValue(rowIndex-1, colIndex-1));
- }
- else if(rowIndex >= 0) {
- // Print a row
- //for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- for(int j=0; j<min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS); j++) {
- System.out.format("%.4f\t", mb.quickGetValue(rowIndex-1, j));
- }
- System.out.println();
- if (mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: the row of given DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d column(s) is being displayed.\n",
- min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- //}
- }
- else if(colIndex >= 0) {
- // Print a column
- for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- System.out.format("%.4f\t", mb.quickGetValue(i, colIndex-1));
- System.out.println();
- }
- if (mb.getNumRows() > DISPLAY_MAX_ROWS) {
- System.out.format("WARNING: the column of given DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS));
- }
- }
- }
-
- /**
- * Prepare current instruction for printing
- * by removing internal delimiters.
- * @param inst Instruction to be displayed
- * @return Post-processed instruction in string format
- */
- private static String prepareInstruction(String inst) {
- String tmp = inst;
- tmp = tmp.replaceAll(Lop.OPERAND_DELIMITOR, " ");
- tmp = tmp.replaceAll(Lop.DATATYPE_PREFIX, ".");
- tmp = tmp.replaceAll(Lop.INSTRUCTION_DELIMITOR, ", ");
-
- return tmp;
- }
-}