You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/12/06 21:00:46 UTC
[1/2] incubator-systemml git commit: [SYSTEMML-1132] Remove unused
classes from project
Repository: incubator-systemml
Updated Branches:
refs/heads/master 8c069ed8e -> 11381c129
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/util/RandN.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/RandN.java b/src/main/java/org/apache/sysml/runtime/util/RandN.java
deleted file mode 100644
index 999676a..0000000
--- a/src/main/java/org/apache/sysml/runtime/util/RandN.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.util;
-
-import java.util.Random;
-
-/**
- * Class that can generate a stream of random numbers from standard
- * normal distribution N(0,1). This class internally makes use of
- * RandNPair, which uses Box-Muller method.
- */
-
-
-public class RandN
-{
- //private long seed;
- Random r;
- private RandNPair pair;
- boolean flag = false; // we use pair.N1 if flag=false, and pair.N2 otherwise
-
- public RandN(long s) {
- init(new Random(s));
- }
-
- public RandN(Random random) {
- init(random);
- }
-
- private void init(Random random) {
- //seed = s;
- r = random;
- pair = new RandNPair();
- flag = false;
- pair.compute(r);
- }
-
- public double nextDouble() {
- double d;
- if (!flag) {
- d = pair.getFirst();
- }
- else {
- d = pair.getSecond();
- pair.compute(r);
- }
- flag = !flag;
- return d;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/utils/Timer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Timer.java b/src/main/java/org/apache/sysml/utils/Timer.java
deleted file mode 100644
index 0d0a709..0000000
--- a/src/main/java/org/apache/sysml/utils/Timer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.utils;
-
-public class Timer
-{
-
-
- long start;
- double sofar;
-
- public Timer() {
- start = 0 ;
- sofar = 0.0;
- }
-
- public void start() {
- start = System.nanoTime();
- sofar = 0.0;
- }
-
- public double stop() {
- double duration = sofar + (System.nanoTime()-start)*1e-6;
- sofar = 0.0;
- start = 0;
- return duration;
- }
-
- public double nanostop() {
- double duration = sofar + (System.nanoTime()-start);
- sofar = 0.0;
- start = 0;
- return duration;
- }
-
- public double pause() {
- sofar += (System.nanoTime()-start)*1e-6;
- return sofar;
- }
-
- public void resume() {
- start = System.nanoTime();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
index 416a615..2b7886f 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java
@@ -20,6 +20,10 @@
package org.apache.sysml.yarn.ropt;
+/**
+ * Experimental feature not used in production.
+ *
+ */
public class ResourceOptimizerCPMigration
{
[2/2] incubator-systemml git commit: [SYSTEMML-1132] Remove unused
classes from project
Posted by de...@apache.org.
[SYSTEMML-1132] Remove unused classes from project
Remove 26 unused Java files from project.
Closes #298.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/11381c12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/11381c12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/11381c12
Branch: refs/heads/master
Commit: 11381c12919110334504519d4de0113ba58aac12
Parents: 8c069ed
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Tue Dec 6 12:15:13 2016 -0800
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Tue Dec 6 12:15:13 2016 -0800
----------------------------------------------------------------------
.../org/apache/sysml/debug/DMLDisassemble.java | 203 -------------
.../controlprogram/parfor/opt/MemoTable.java | 128 --------
.../parfor/opt/MemoTableEntry.java | 110 -------
.../sysml/runtime/functionobjects/MaxIndex.java | 77 -----
.../sysml/runtime/functionobjects/MinIndex.java | 77 -----
.../runtime/instructions/cp/FileObject.java | 71 -----
.../instructions/spark/data/CountLinesInfo.java | 40 ---
.../ConvertColumnRDDToBinaryBlock.java | 70 -----
.../ConvertMLLibBlocksToBinaryBlocks.java | 79 -----
.../ConvertTextLineToBinaryCellFunction.java | 93 ------
.../spark/functions/ConvertTextToString.java | 33 --
.../FindMatrixBlockFromMatrixIndexes.java | 46 ---
.../spark/functions/GetMLLibBlocks.java | 123 --------
.../spark/functions/LastCellInMatrixBlock.java | 55 ----
.../functions/MatrixVectorBinaryOpFunction.java | 65 ----
.../data/PoissonRandomMatrixGenerator.java | 51 ----
.../sysml/runtime/matrix/data/TaggedInt.java | 61 ----
.../runtime/matrix/data/TaggedPartialBlock.java | 35 ---
.../data/WeightedPairToSortInputConverter.java | 60 ----
.../sysml/runtime/matrix/mapred/CachedMap.java | 135 --------
.../runtime/matrix/mapred/CachedMapElement.java | 29 --
.../runtime/matrix/mapred/MMCJMRCombiner.java | 55 ----
.../runtime/matrix/mapred/MMCJMRReducer.java | 304 -------------------
.../sort/CompactDoubleIntInputFormat.java | 213 -------------
.../runtime/util/BinaryBlockInputFormat.java | 6 +-
.../org/apache/sysml/runtime/util/RandN.java | 67 ----
src/main/java/org/apache/sysml/utils/Timer.java | 62 ----
.../yarn/ropt/ResourceOptimizerCPMigration.java | 4 +
28 files changed, 8 insertions(+), 2344 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDisassemble.java b/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
deleted file mode 100644
index 50145e6..0000000
--- a/src/main/java/org/apache/sysml/debug/DMLDisassemble.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.debug;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
-import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
-import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
-import org.apache.sysml.runtime.controlprogram.Program;
-import org.apache.sysml.runtime.controlprogram.ProgramBlock;
-import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
-import org.apache.sysml.runtime.instructions.Instruction;
-import org.apache.sysml.runtime.instructions.MRJobInstruction;
-import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
-import org.apache.sysml.runtime.instructions.cp.CPInstruction;
-
-
-/**
- * Class to produce mapping between source file lines and runtime instructions
- */
-public class DMLDisassemble
-{
- private Program _prog;
- //Map between DML program line numbers and corresponding runtime instruction(s)
- private TreeMap<Integer, ArrayList<Instruction>> DMLInstMap;
-
- /**
- * Constructor for DML disassembler class
- * @param prog Program reference (e.g., function repository)
- */
- public DMLDisassemble(Program prog)
- {
- _prog = prog;
- DMLInstMap = new TreeMap<Integer, ArrayList<Instruction>>();
- }
-
- ////////////////////////////////////////////////
- // getter and setter functionality
- ////////////////////////////////////////////////
-
- /**
- * Getter for program reference field
- * @return _prog Program reference
- */
- public Program getProgram()
- {
- return _prog;
- }
-
- /**
- * Getter for DML script to instruction map field
- * @return DMLInstMap Map between DML script line numbers and runtime instruction(s)
- */
- public TreeMap<Integer, ArrayList<Instruction>> getDMLInstMap()
- {
- return DMLInstMap;
- }
-
- /**
- * Setter for program reference field
- * @param prog Program reference
- */
- public void setProgram(Program prog)
- {
- this._prog = prog;
- }
-
- /**
- * Setter for DML instruction map field (i.e. disassembler functionality)
- * For each DML script line, get runtime instructions (if any)
- */
- public void setDMLInstMap()
- {
- DMLInstMap = new TreeMap<Integer, ArrayList<Instruction>>();
- if (_prog != null)
- {
- //Functions: For each function program block (if any), get instructions corresponding to each line number
- HashMap<String,FunctionProgramBlock> funcMap = this._prog.getFunctionProgramBlocks();
- if (funcMap != null && !funcMap.isEmpty() )
- {
- for (Entry<String, FunctionProgramBlock> e : funcMap.entrySet())
- {
- String fkey = e.getKey();
- FunctionProgramBlock fpb = e.getValue();
- if(fpb instanceof ExternalFunctionProgramBlock)
- System.err.println("----EXTERNAL FUNCTION "+fkey+"\n");
- else
- {
- System.err.println("----FUNCTION "+fkey+"\n");
- for (ProgramBlock pb : fpb.getChildBlocks())
- setProgramBlockInstMap(pb);
- }
- }
- }
- //Main program: for each program block, get instructions corresponding to current line number (if any)
- for (ProgramBlock pb : this._prog.getProgramBlocks())
- {
- if (pb != null)
- setProgramBlockInstMap(pb);
- }
- }
- }
-
- /**
- * For each program block, get runtime instructions (if any)
- * @param pb Current program block
- */
- private void setProgramBlockInstMap(ProgramBlock pb) {
- if (pb instanceof FunctionProgramBlock)
- {
- FunctionProgramBlock fpb = (FunctionProgramBlock)pb;
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock) pb;
- this.setInstMap(wpb.getPredicate());
- for( ProgramBlock pbc : wpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock ipb = (IfProgramBlock) pb;
- this.setInstMap(ipb.getPredicate());
- for( ProgramBlock pbc : ipb.getChildBlocksIfBody() )
- setProgramBlockInstMap(pbc);
- if( !ipb.getChildBlocksElseBody().isEmpty() )
- {
- for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
- setProgramBlockInstMap(pbc);
- }
- }
- else if (pb instanceof ForProgramBlock) //incl parfor
- {
- ForProgramBlock fpb = (ForProgramBlock) pb;
- this.setInstMap(fpb.getFromInstructions());
- this.setInstMap(fpb.getToInstructions());
- this.setInstMap(fpb.getIncrementInstructions());
- for( ProgramBlock pbc : fpb.getChildBlocks() )
- setProgramBlockInstMap(pbc);
- }
- else
- {
- this.setInstMap(pb.getInstructions());
- }
- }
-
- /**
- * For each instruction, generate map with corresponding DML
- * script line number
- * @param instructions Instructions for current program block
- */
- private void setInstMap(ArrayList<Instruction> instructions ) {
- for (int i = 0; i < instructions.size() ; i++) {
- Instruction currInst = instructions.get(i);
- if (currInst instanceof MRJobInstruction) {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- for (Integer lineNumber : currMRInst.getMRJobInstructionsLineNumbers()) {
- if (!DMLInstMap.containsKey(lineNumber))
- DMLInstMap.put(lineNumber, new ArrayList<Instruction>());
- DMLInstMap.get(lineNumber).add(currMRInst);
- }
- }
- else if (currInst instanceof CPInstruction) {
- CPInstruction currCPInst = (CPInstruction) currInst;
- // Check if current instruction line number correspond to source line number
- if (!DMLInstMap.containsKey(currInst.getLineNum()))
- DMLInstMap.put(currInst.getLineNum(), new ArrayList<Instruction>());
- DMLInstMap.get(currInst.getLineNum()).add(currCPInst);
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- // Check if current instruction line number correspond to source line number
- if (!DMLInstMap.containsKey(currInst.getLineNum()))
- DMLInstMap.put(currInst.getLineNum(), new ArrayList<Instruction>());
- DMLInstMap.get(currInst.getLineNum()).add(currBPInst);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
deleted file mode 100644
index 421037c..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTable.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-
-import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestMeasure;
-
-public class MemoTable
-{
-
-
- //logical plan node, list of physical plan nodes
- private Map<Long,Collection<MemoTableEntry>> _memo;
-
- public MemoTable( )
- {
- _memo = new HashMap<Long, Collection<MemoTableEntry>>();
- }
-
- public void putMemoTableEntry( long ID, MemoTableEntry e, boolean keepOnlyMin )
- {
- //create memo structure on demand
- Collection<MemoTableEntry> entries = _memo.get(ID);
- if( entries == null )
- {
- entries = new LinkedList<MemoTableEntry>();
- _memo.put(ID, entries);
- }
-
- //add the respective entry
- if( keepOnlyMin )
- {
- if( !entries.isEmpty() )
- {
- MemoTableEntry old = entries.iterator().next();
- if( e.getEstLTime()<old.getEstLTime() )
- {
- entries.remove(old);
- entries.add(e);
- }
- }
- else
- entries.add(e);
- }
- else
- entries.add(e);
- }
-
- public boolean hasCandidates()
- {
- for( Collection<MemoTableEntry> entries : _memo.values() )
- if( entries != null && !entries.isEmpty() )
- return true;
- return false;
- }
-
- public Collection<MemoTableEntry> getCandidates()
- {
- Collection<MemoTableEntry> C = new LinkedList<MemoTableEntry>();
-
- for( Collection<MemoTableEntry> entries : _memo.values() )
- if( entries != null && !entries.isEmpty() )
- C.addAll(entries);
-
- return C;
- }
-
- public MemoTableEntry getMinTimeEntry( long ID )
- {
- return getMin( ID, TestMeasure.EXEC_TIME );
- }
-
- public MemoTableEntry getMinMemEntry( long ID )
- {
- return getMin( ID, TestMeasure.MEMORY_USAGE );
- }
-
- private MemoTableEntry getMin( long ID, TestMeasure measure )
- {
- MemoTableEntry minObj = null;
- double minVal = Double.MAX_VALUE;
-
- Collection<MemoTableEntry> entries = _memo.get( ID );
- if( entries != null )
- for( MemoTableEntry e : entries )
- switch( measure )
- {
- case EXEC_TIME:
- if( e.getEstLTime() < minVal )
- {
- minObj = e;
- minVal = e.getEstLTime();
- }
- break;
- case MEMORY_USAGE:
- if( e.getEstLMemory() < minVal )
- {
- minObj = e;
- minVal = e.getEstLMemory();
- }
- break;
- }
-
- return minObj;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
deleted file mode 100644
index f9fa208..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/MemoTableEntry.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import org.apache.sysml.runtime.controlprogram.ProgramBlock;
-
-public class MemoTableEntry
-{
-
-
- //physical plan representations
- private long _id = -1;
- private OptNode _rtOptNode = null;
- private ProgramBlock _rtPB = null;
-
- //statistics
- private double _estLMemory = -1;
- private double _estLTime = -1;
-
- public MemoTableEntry( long id, OptNode rtnode, ProgramBlock pb, double lm, double lt )
- {
- _id = id;
- _rtOptNode = rtnode;
- _rtPB = pb;
- _estLMemory = lm;
- _estLTime = lt;
- }
-
- public long getID()
- {
- return _id;
- }
-
- public void setID( long id )
- {
- _id = id;
- }
-
- public OptNode getRtOptNode()
- {
- return _rtOptNode;
- }
-
- public void setRtOptNode(OptNode rtOptNode)
- {
- _rtOptNode = rtOptNode;
- }
-
- public ProgramBlock getRtProgramBlock()
- {
- return _rtPB;
- }
-
- public void setRtProgramBlock( ProgramBlock pb )
- {
- _rtPB = pb;
- }
-
- public double getEstLMemory()
- {
- return _estLMemory;
- }
-
- public void setEstLMemory(double estLMemory)
- {
- _estLMemory = estLMemory;
- }
-
- public double getEstLTime()
- {
- return _estLTime;
- }
-
- public void setEstLTime(double estLTime)
- {
- _estLTime = estLTime;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("ID = ");
- sb.append(_id);
- sb.append("RtOptNode = ");
- sb.append(_rtOptNode.explain(0, false));
- sb.append("RtProg = ");
- sb.append(_rtPB.getClass().toString());
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
deleted file mode 100644
index 101c872..0000000
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/MaxIndex.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.functionobjects;
-
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
-
-
-public class MaxIndex extends IndexFunction
-{
-
- private static final long serialVersionUID = -4941564912238185729L;
-
- private static MaxIndex singleObj = null;
-
- private MaxIndex() {
- // nothing to do here
- }
-
- public static MaxIndex getMaxIndexFnObject() {
- if ( singleObj == null )
- singleObj = new MaxIndex();
- return singleObj;
- }
-
- public Object clone() throws CloneNotSupportedException {
- // cloning is not supported for singleton classes
- throw new CloneNotSupportedException();
- }
-
- @Override
- public void execute(MatrixIndexes in, MatrixIndexes out) {
- long max = Math.max(in.getRowIndex(), in.getColumnIndex());
- out.setIndexes(max, max);
- }
-
- @Override
- public void execute(CellIndex in, CellIndex out) {
- int max = Math.max(in.row, in.column);
- out.set(max, max);
- }
-
- @Override
- public boolean computeDimension(int row, int col, CellIndex retDim) {
- int max=Math.max(row, col);
- retDim.set(max, max);
- return false;
- }
-
- public boolean computeDimension(MatrixCharacteristics in, MatrixCharacteristics out) throws DMLRuntimeException
- {
- long maxMatrix=Math.max(in.getRows(), in.getCols());
- int maxBlock=Math.max(in.getRowsPerBlock(), in.getColsPerBlock());
- out.set(maxMatrix, maxMatrix, maxBlock, maxBlock);
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
deleted file mode 100644
index b14bf87..0000000
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/MinIndex.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.functionobjects;
-
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
-
-
-public class MinIndex extends IndexFunction
-{
-
- private static final long serialVersionUID = -4159274805822230421L;
-
- private static MinIndex singleObj = null;
-
- private MinIndex() {
- // nothing to do here
- }
-
- public static MinIndex getMinIndexFnObject() {
- if ( singleObj == null )
- singleObj = new MinIndex();
- return singleObj;
- }
-
- public Object clone() throws CloneNotSupportedException {
- // cloning is not supported for singleton classes
- throw new CloneNotSupportedException();
- }
-
- @Override
- public void execute(MatrixIndexes in, MatrixIndexes out) {
- long min = Math.min(in.getRowIndex(), in.getColumnIndex());
- out.setIndexes(min, min);
- }
-
- @Override
- public void execute(CellIndex in, CellIndex out) {
- int min = Math.min(in.row, in.column);
- out.set(min, min);
- }
-
- @Override
- public boolean computeDimension(int row, int col, CellIndex retDim) {
- int min=Math.min(row, col);
- retDim.set(min, min);
- return false;
- }
-
- public boolean computeDimension(MatrixCharacteristics in, MatrixCharacteristics out) throws DMLRuntimeException
- {
- long minMatrix=Math.min(in.getRows(), in.getCols());
- int minBlock=Math.min(in.getRowsPerBlock(), in.getColsPerBlock());
- out.set(minMatrix, minMatrix, minBlock, minBlock);
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
deleted file mode 100644
index aab0e0c..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FileObject.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.cp;
-
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-
-
-public class FileObject extends Data
-{
-
- private static final long serialVersionUID = 2057548889127080668L;
-
- public String toString() {
- return "(" + _name + "," + _filePath + ")" ;
- }
-
- private String _name;
- private String _filePath;
- private boolean _isLocal;
-
- public FileObject(String path){
- this(null,path);
- }
-
- public FileObject(String name, String path){
- super(DataType.SCALAR, ValueType.STRING);
- _name = name;
- _filePath = path;
- _isLocal = false;
- }
-
- public void setLocal(){
- _isLocal = true;
- }
-
- public String getName(){
- return _name;
- }
-
- public String getFilePath(){
- return _filePath;
- }
-
- public boolean isLocal(){
- return _isLocal;
- }
-
- @Override
- public String getDebugName() {
- // TODO Auto-generated method stub
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
deleted file mode 100644
index b1614a4..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/CountLinesInfo.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sysml.runtime.instructions.spark.data;
-
-import java.io.Serializable;
-
-public class CountLinesInfo implements Serializable {
- private static final long serialVersionUID = 4178309746487858987L;
- private long numLines;
- private long expectedNumColumns;
- public long getNumLines() {
- return numLines;
- }
- public void setNumLines(long numLines) {
- this.numLines = numLines;
- }
- public long getExpectedNumColumns() {
- return expectedNumColumns;
- }
- public void setExpectedNumColumns(long expectedNumColumns) {
- this.expectedNumColumns = expectedNumColumns;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
deleted file mode 100644
index 90c3e66..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertColumnRDDToBinaryBlock.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.util.UtilFunctions;
-
-public class ConvertColumnRDDToBinaryBlock {
-
- public JavaPairRDD<MatrixIndexes,MatrixBlock> getBinaryBlockedRDD(JavaRDD<Double> columnRDD, int brlen, long rlen, SparkExecutionContext sec) {
-
- // Since getRowOffsets requires a collect to get linecount, might as well do a collect here and convert it to RDD as CP operation.
- // This limitation also exists for CSVReblock
- // TODO: Optimize this for large column vectors !!!
-
- List<Double> values = columnRDD.collect();
-
- int i = 0; int brIndex = 1;
- // ------------------------------------------------------------------
- // Compute local block size:
- int lrlen = UtilFunctions.computeBlockSize(rlen, brIndex, brlen);
- // ------------------------------------------------------------------
- ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> retVal = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>();
- MatrixBlock blk = new MatrixBlock(lrlen, 1, true);
- for(Double value : values) {
- if(i == lrlen) {
- // Block filled in. Create new block.
- retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(brIndex, 1), blk));
- brIndex++;
- lrlen = UtilFunctions.computeBlockSize(rlen, brIndex, brlen);
- blk = new MatrixBlock(lrlen, 1, true);
- i = 0;
- }
- blk.appendValue(i, 0, value);
- // blk.setValue(i, 1, value);
- i++;
- }
-
- // Now insert last block
- retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(brIndex, 1), blk));
-
- return JavaPairRDD.fromJavaRDD(sec.getSparkContext().parallelize(retVal));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
deleted file mode 100644
index 2210bbb..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertMLLibBlocksToBinaryBlocks.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.mllib.linalg.DenseMatrix;
-import org.apache.spark.mllib.linalg.Matrix;
-import org.apache.spark.mllib.linalg.SparseMatrix;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.util.UtilFunctions;
-
-public class ConvertMLLibBlocksToBinaryBlocks implements PairFunction<Tuple2<Tuple2<Object,Object>,Matrix>, MatrixIndexes, MatrixBlock> {
-
- private static final long serialVersionUID = 1876492711549091662L;
-
- long rlen; long clen; int brlen; int bclen;
- public ConvertMLLibBlocksToBinaryBlocks(long rlen, long clen, int brlen, int bclen) {
- this.rlen = rlen;
- this.clen = clen;
- this.brlen = brlen;
- this.bclen = bclen;
- }
-
- @Override
- public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<Tuple2<Object, Object>, Matrix> kv) throws Exception {
- long nRows = (Long) kv._1._1;
- long nCols = (Long) kv._1._2;
- // ------------------------------------------------------------------
- // Compute local block size:
- int lrlen = UtilFunctions.computeBlockSize(rlen, nRows, brlen);
- int lclen = UtilFunctions.computeBlockSize(clen, nCols, bclen);
- // ------------------------------------------------------------------
- MatrixBlock blk = null;
- double [] vals = null;
- if(kv._2 instanceof DenseMatrix) {
- blk = new MatrixBlock(lrlen, lclen, false);
- vals = ((DenseMatrix) kv._2).values();
- }
- else if(kv._2 instanceof SparseMatrix) {
- blk = new MatrixBlock(lrlen, lclen, true);
- vals = ((SparseMatrix) kv._2).values();
- }
- else {
- throw new Exception("Unsupported type of matrix");
- }
-
- int iter = 0;
- for(int i = 0; i < lrlen-1; i++) {
- for(int j = 0; j < lclen-1; j++) {
- if(vals[iter] != 0)
- blk.appendValue(i, j, vals[iter]);
- iter++;
- }
- }
-
- return new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(nRows, nCols), blk);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
deleted file mode 100644
index b135497..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.hadoop.io.Text;
-import org.apache.spark.api.java.function.PairFunction;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.data.Converter;
-import org.apache.sysml.runtime.matrix.data.MatrixCell;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.MatrixValue;
-import org.apache.sysml.runtime.matrix.data.Pair;
-import org.apache.sysml.runtime.matrix.data.TextToBinaryCellConverter;
-import org.apache.sysml.runtime.util.UtilFunctions;
-
-
-public class ConvertTextLineToBinaryCellFunction implements PairFunction<String, MatrixIndexes, MatrixCell> {
-
- private static final long serialVersionUID = -3672377410407066396L;
- private int brlen;
- private int bclen;
- private long rlen;
- private long clen;
-
- public ConvertTextLineToBinaryCellFunction(MatrixCharacteristics mcOut) {
- this.brlen = mcOut.getRowsPerBlock();
- this.bclen = mcOut.getColsPerBlock();
- this.rlen = mcOut.getRows();
- this.clen = mcOut.getCols();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Tuple2<MatrixIndexes, MatrixCell> call(String line) throws Exception {
- @SuppressWarnings("rawtypes")
- Converter converter = new TextToBinaryCellConverter();
- converter.setBlockSize(brlen, bclen);
- converter.convert(null, new Text(line));
-
- Pair<MatrixIndexes, MatrixValue> retVal = null;
- if(converter.hasNext()) {
- retVal = converter.next();
-
- if(retVal.getKey().getRowIndex() > rlen || retVal.getKey().getColumnIndex() > clen) {
- throw new Exception("Either incorrect metadata provided to text reblock (" + rlen + "," + clen
- + ") or incorrect input line:" + line);
- }
- // ------------------------------------------------------------------------------------------
- // Get appropriate indexes for blockIndexes and cell
- // For line: 1020 704 2.362153706180234 (assuming default block size: 1000 X 1000),
- // blockRowIndex = 2, blockColIndex = 1, rowIndexInBlock = 19, colIndexInBlock = 703 ... TODO: double check this !!!
- long blockRowIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getRowIndex(), (int) brlen);
- long blockColIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getColumnIndex(), (int) bclen);
- long rowIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getRowIndex(), brlen);
- long colIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getColumnIndex(), bclen);
- // Perform sanity check
- if(blockRowIndex <= 0 || blockColIndex <= 0 || rowIndexInBlock < 0 || colIndexInBlock < 0) {
- throw new Exception("Error computing indexes for the line:" + line);
- }
- // ------------------------------------------------------------------------------------------
-
- MatrixIndexes blockIndexes = new MatrixIndexes(blockRowIndex, blockColIndex);
- MatrixCell cell = new MatrixCell(((MatrixCell)retVal.getValue()).getValue());
-
- return new Tuple2<MatrixIndexes, MatrixCell>(blockIndexes, cell);
- }
-
- // In case of header for matrix format
- return new Tuple2<MatrixIndexes, MatrixCell>(new MatrixIndexes(-1, -1), null);
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
deleted file mode 100644
index 5050ba2..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextToString.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.hadoop.io.Text;
-import org.apache.spark.api.java.function.Function;
-
-public class ConvertTextToString implements Function<Text, String> {
-
- private static final long serialVersionUID = 3916028932406746166L;
-
- @Override
- public String call(Text arg0) throws Exception {
- return arg0.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
deleted file mode 100644
index 57cebcb..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/FindMatrixBlockFromMatrixIndexes.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.spark.api.java.function.Function;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-
-/**
- * Function class find the tuple that matches the given MatrixIndexes.
- */
-public class FindMatrixBlockFromMatrixIndexes implements Function<Tuple2<MatrixIndexes,MatrixBlock>, Boolean>
-{
- private static final long serialVersionUID = 3579786335136549745L;
- MatrixIndexes _key = null;
-
- public FindMatrixBlockFromMatrixIndexes(MatrixIndexes k) {
- _key = k;
- }
-
- @Override
- public Boolean call(Tuple2<MatrixIndexes, MatrixBlock> v1)
- throws Exception {
- return (v1._1().compareTo(_key) == 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
deleted file mode 100644
index 8601368..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-
-import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.mllib.linalg.DenseMatrix;
-import org.apache.spark.mllib.linalg.Matrix;
-import org.apache.spark.mllib.linalg.SparseMatrix;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.matrix.data.IJV;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.util.UtilFunctions;
-
-public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>, Tuple2<Object, Object>, Matrix> {
-
- private static final long serialVersionUID = 5977179886312384315L;
-
- long rlen; long clen; int brlen; int bclen;
- public GetMLLibBlocks(long rlen, long clen, int brlen, int bclen) {
- this.rlen = rlen;
- this.clen = clen;
- this.brlen = brlen;
- this.bclen = bclen;
- }
-
- private int getNNZ(MatrixBlock blk) {
- if(blk.getNonZeros() != -1) {
- return (int) blk.getNonZeros();
- }
- else if(blk.isInSparseFormat()) {
- Iterator<IJV> iter = blk.getSparseBlockIterator();
- int nnz = 0;
- while( iter.hasNext() ) {
- nnz++;
- }
- return nnz;
- }
- else {
- return blk.getDenseBlock().length;
- }
- }
-
- private int [] getArray(ArrayList<Integer> al) {
- int [] retVal = new int[al.size()];
- int i = 0;
- for(Integer v : al) {
- retVal[i] = v;
- i++;
- }
- return retVal;
- }
-
- @Override
- public Tuple2<Tuple2<Object, Object>, Matrix> call(
- Tuple2<MatrixIndexes, MatrixBlock> kv) throws Exception {
- Integer blockRowIndex = (int) kv._1.getRowIndex();
- Integer blockColIndex = (int) kv._1.getColumnIndex();
-
- Matrix mllibBlock = null;
- MatrixBlock blk = kv._2;
- // ------------------------------------------------------------------
- // Compute local block size:
- int lrlen = UtilFunctions.computeBlockSize(rlen, kv._1.getRowIndex(), brlen);
- int lclen = UtilFunctions.computeBlockSize(clen, kv._1.getColumnIndex(), bclen);
- // ------------------------------------------------------------------
-
- if(blk.isInSparseFormat()) {
- Iterator<IJV> iter = blk.getSparseBlockIterator();
- int nnz = getNNZ(blk);
- double [] values = new double[nnz];
- int [] rowIndices = new int[nnz];
- ArrayList<Integer> colPtrList = new ArrayList<Integer>();
- int lastCol = -1; int index = 0;
- while( iter.hasNext() ) {
- IJV cell = iter.next(); // TODO: This might output global
- // MLLib's sparse rows are stored as Compressed Sparse Column (CSC) format
- if(lastCol != cell.getJ()) {
- lastCol = cell.getJ();
- colPtrList.add(lastCol);
- }
- try {
- values[index] = cell.getV();
- rowIndices[index] = cell.getI();
- index++;
- }
- catch(Exception e) {
- throw new Exception("The number of non-zeros are not set correctly.");
- }
- }
-
- int [] colPtrs = getArray(colPtrList);
- mllibBlock = new SparseMatrix(lrlen, lclen, colPtrs, rowIndices, values);
- }
- else {
- mllibBlock = new DenseMatrix(lrlen, lclen, blk.getDenseBlock());
- }
- return new Tuple2<Tuple2<Object,Object>, Matrix>(new Tuple2<Object,Object>(blockRowIndex, blockColIndex), mllibBlock);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
deleted file mode 100644
index ebef207..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/LastCellInMatrixBlock.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.spark.api.java.function.Function;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-
-/**
- * Function class that finds the index of last cell in the given matrix block.
- */
-public class LastCellInMatrixBlock implements Function<Tuple2<MatrixIndexes, MatrixBlock>, MatrixIndexes> {
-
- private static final long serialVersionUID = -4930522353195801521L;
-
- int _rpb, _cpb;
-
- public LastCellInMatrixBlock(int rpb, int cpb) {
- _rpb = rpb;
- _cpb = cpb;
- }
-
- @Override
- public MatrixIndexes call(Tuple2<MatrixIndexes, MatrixBlock> v1)
- throws Exception {
-
- // Find the indices of last cell in the block
- long r = _rpb * (v1._1().getRowIndex()-1) + v1._2().getNumRows();
- long c = _cpb * (v1._1().getColumnIndex()-1) + v1._2().getNumColumns();
-
- return new MatrixIndexes(r, c);
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
deleted file mode 100644
index bc6035d..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.functions;
-
-import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.broadcast.Broadcast;
-
-import scala.Tuple2;
-
-import org.apache.sysml.lops.BinaryM.VectorType;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
-
-public class MatrixVectorBinaryOpFunction implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes,MatrixBlock>
-{
-
- private static final long serialVersionUID = -7695883019452417300L;
-
- private BinaryOperator _op = null;
- private Broadcast<PartitionedBlock<MatrixBlock>> _pmV = null;
- private VectorType _vtype = null;
-
- public MatrixVectorBinaryOpFunction( BinaryOperator op, Broadcast<PartitionedBlock<MatrixBlock>> binput, VectorType vtype )
- {
- _op = op;
- _pmV = binput;
- _vtype = vtype;
- }
-
- @Override
- public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
- throws Exception
- {
- MatrixIndexes ix = arg0._1();
- MatrixBlock in1 = arg0._2();
-
- //get the rhs block
- int rix= (int)((_vtype==VectorType.COL_VECTOR) ? ix.getRowIndex() : 1);
- int cix= (int)((_vtype==VectorType.COL_VECTOR) ? 1 : ix.getColumnIndex());
- MatrixBlock in2 = _pmV.value().getBlock(rix, cix);
-
- //execute the binary operation
- MatrixBlock ret = (MatrixBlock) (in1.binaryOperations (_op, in2, new MatrixBlock()));
- return new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(ix), ret);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java b/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
deleted file mode 100644
index 97fd32a..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/PoissonRandomMatrixGenerator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.matrix.data;
-
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.util.PoissonPRNGenerator;
-
-public class PoissonRandomMatrixGenerator extends RandomMatrixGenerator {
-
- private double _mean=1.0;
-
- PoissonRandomMatrixGenerator(String pdf, int r, int c, int rpb, int cpb, double sp, double mean) throws DMLRuntimeException
- {
- _mean = mean;
- init(pdf, r, c, rpb, cpb, sp, Double.NaN, Double.NaN);
- setupValuePRNG();
- }
-
- @Override
- protected void setupValuePRNG() throws DMLRuntimeException
- {
- if(_pdf.equalsIgnoreCase(LibMatrixDatagen.RAND_PDF_POISSON))
- {
- if(_mean <= 0)
- throw new DMLRuntimeException("Invalid parameter (" + _mean + ") for Poisson distribution.");
-
- _valuePRNG = new PoissonPRNGenerator(_mean);
- }
- else
- throw new DMLRuntimeException("Expecting a Poisson distribution (pdf = " + _pdf);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
deleted file mode 100644
index 62307d0..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedInt.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.data;
-
-import org.apache.hadoop.io.IntWritable;
-
-public class TaggedInt extends Tagged<IntWritable>
-{
-
- public TaggedInt()
- {
- tag=-1;
- base=new IntWritable();
- }
-
- public TaggedInt(IntWritable b, byte t) {
- super(b, t);
- }
-
- public int hashCode()
- {
- return base.hashCode()+tag;
- }
-
- public int compareTo(TaggedInt other)
- {
- if(this.tag!=other.tag)
- return (this.tag-other.tag);
- else if(this.base.get()!=other.base.get())
- return (this.base.get()-other.base.get());
- return 0;
- }
-
- @Override
- public boolean equals(Object other)
- {
- if( !(other instanceof TaggedInt))
- return false;
-
- TaggedInt tother = (TaggedInt)other;
- return (this.tag==tother.tag && this.base.get()==tother.base.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
deleted file mode 100644
index 6055c44..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedPartialBlock.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.data;
-
-public class TaggedPartialBlock extends Tagged<PartialBlock>
-{
-
- public TaggedPartialBlock(PartialBlock b, byte t) {
- super(b, t);
- }
-
- public TaggedPartialBlock()
- {
- tag=-1;
- base=new PartialBlock();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
deleted file mode 100644
index f618585..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPairToSortInputConverter.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.data;
-
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.IntWritable;
-
-public class WeightedPairToSortInputConverter implements
-Converter<MatrixIndexes, WeightedPair, DoubleWritable, IntWritable>
-{
-
- private DoubleWritable outKey=new DoubleWritable();
- private IntWritable outValue=new IntWritable();
- private Pair<DoubleWritable, IntWritable> pair=new Pair<DoubleWritable, IntWritable>(outKey, outValue);
- private boolean hasValue=false;
- @Override
- public void convert(MatrixIndexes k1, WeightedPair v1) {
- outKey.set(v1.getValue());
- outValue.set((int)v1.getWeight());
- hasValue=true;
- }
-
- @Override
- public boolean hasNext() {
- return hasValue;
- }
-
- @Override
- public Pair<DoubleWritable, IntWritable> next() {
- if(!hasValue)
- return null;
-
- hasValue=false;
- return pair;
- }
-
- @Override
- public void setBlockSize(int rl, int cl) {
- //DO nothing
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
deleted file mode 100644
index ce04ca1..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMap.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.mapred;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Set;
-
-
-public class CachedMap<T extends CachedMapElement>
-{
-
-
- protected HashMap<Byte, ArrayList<Integer>> map=new HashMap<Byte, ArrayList<Integer>>();
- protected ArrayList<T> cache=new ArrayList<T>();
- protected int numValid=0;
- protected ArrayList<T> returnListCache=new ArrayList<T>(4);
-
- public CachedMap()
- {}
-
- @SuppressWarnings("unchecked")
- public T add(Byte index, T value)
- {
- if(numValid<cache.size())
- cache.get(numValid).set(value);
- else
- cache.add((T) value.duplicate());
- ArrayList<Integer> list=map.get(index);
- if(list==null)
- {
- list=new ArrayList<Integer>(4);
- map.put(index, list);
- }
- list.add(numValid);
- numValid++;
- return cache.get(numValid-1);
- }
-
- public void reset()
- {
- numValid=0;
- map.clear();
- }
-
- public void remove(byte index)
- {
- ArrayList<Integer> list=map.remove(index);
- if(list==null)
- return;
-
- for(Integer cacheIndex: list)
- {
- if(cacheIndex==numValid-1)
- {
- numValid--;
- return;
- }
- //swap the last element and the element to remove
- T lastElem=cache.get(numValid-1);
- cache.set(numValid-1, cache.get(cacheIndex));
- cache.set(cacheIndex, lastElem);
- //remap the indexes
- for(ArrayList<Integer> lst: map.values())
- for(int i=0; i<lst.size(); i++)
- {
- if(lst.get(i)==numValid-1)
- {
- lst.set(i, cacheIndex);
- break;
- }
- }
- numValid--;
- }
- }
-
- public ArrayList<T> get(byte index)
- {
- ArrayList<Integer> list=map.get(index);
- if(list==null)
- return null;
-
- returnListCache.clear();
- for(Integer i: list)
- returnListCache.add(cache.get(i));
- return returnListCache;
- }
-
- public T getFirst(byte index)
- {
- ArrayList<Integer> list=map.get(index);
- if(list!=null && !list.isEmpty())
- return cache.get(list.get(0));
- else
- return null;
- }
-
- public Set<Byte> getIndexesOfAll()
- {
- return map.keySet();
- }
-
-/* public String toString()
- {
- String str="";
- for(Entry<Byte,Integer> e: map.entrySet())
- str+=e.getKey()+" <--> "+cache.get(e.getValue())+"\n";
- return str;
- }*/
-
- public String toString()
- {
- String str="numValid: "+numValid+"\n"+map.toString()+"\n"+cache.toString();
- return str;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
deleted file mode 100644
index 7d6acdc..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedMapElement.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.mapred;
-
-public abstract class CachedMapElement
-{
-
-
- public abstract void set(CachedMapElement elem);
- public abstract CachedMapElement duplicate();
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
deleted file mode 100644
index 6b825fc..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCombiner.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.mapred;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import org.apache.sysml.runtime.matrix.data.MatrixValue;
-import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
-
-
-public class MMCJMRCombiner extends MMCJMRCombinerReducerBase
-implements Reducer<TaggedFirstSecondIndexes, MatrixValue, TaggedFirstSecondIndexes, MatrixValue>
-{
-
- public void reduce(TaggedFirstSecondIndexes indexes, Iterator<MatrixValue> values,
- OutputCollector<TaggedFirstSecondIndexes, MatrixValue> out,
- Reporter report) throws IOException {
-
- //perform aggregate
- MatrixValue aggregateValue=performAggregateInstructions(indexes, values);
-
- if(aggregateValue!=null)
- out.collect(indexes, aggregateValue);
-
- }
-
- public void configure(JobConf job)
- {
- super.configure(job);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
deleted file mode 100644
index a9b7d8b..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRReducer.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.mapred;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.MatrixValue;
-import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
-import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
-import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
-import org.apache.sysml.runtime.util.MapReduceTool;
-
-
-public class MMCJMRReducer extends MMCJMRCombinerReducerBase
-implements Reducer<TaggedFirstSecondIndexes, MatrixValue, Writable, Writable>
-{
-
- private static class RemainIndexValue
- {
- public long remainIndex;
- public MatrixValue value;
- private Class<? extends MatrixValue> valueClass;
- public RemainIndexValue(Class<? extends MatrixValue> cls) throws Exception
- {
- remainIndex=-1;
- valueClass=cls;
- value=valueClass.newInstance();
- }
- public RemainIndexValue(long ind, MatrixValue b) throws Exception
- {
- remainIndex=ind;
- valueClass=b.getClass();
- value=valueClass.newInstance();
- value.copy(b);
- }
- public void set(long ind, MatrixValue b)
- {
- remainIndex=ind;
- value.copy(b);
- }
- }
-
- //in memory cache to hold the records from one input matrix for the cross product
- private ArrayList<RemainIndexValue> cache=new ArrayList<RemainIndexValue>(100);
- private int cacheSize=0;
-
- //to cache output, so that we can do some partial aggregation here
- private long OUT_CACHE_SIZE;
- private HashMap<MatrixIndexes, MatrixValue> outCache;
-
- //variables to keep track of the flow
- private double prevFirstIndex=-1;
- private int prevTag=-1;
-
- //temporary variable
- private MatrixIndexes indexesbuffer=new MatrixIndexes();
- private RemainIndexValue remainingbuffer=null;
- private MatrixValue valueBuffer=null;
-
- private boolean outputDummyRecords = false;
-
- @Override
- public void reduce(TaggedFirstSecondIndexes indexes, Iterator<MatrixValue> values,
- OutputCollector<Writable, Writable> out,
- Reporter report) throws IOException {
- long start=System.currentTimeMillis();
-// LOG.info("---------- key: "+indexes);
-
- commonSetup(report);
-
- //perform aggregate
- MatrixValue aggregateValue=performAggregateInstructions(indexes, values);
-
- if(aggregateValue==null)
- return;
-
- int tag=indexes.getTag();
- long firstIndex=indexes.getFirstIndex();
- long secondIndex=indexes.getSecondIndex();
-
- //for a different k
- if(prevFirstIndex!=firstIndex)
- {
- resetCache();
- prevFirstIndex=firstIndex;
- }else if(prevTag>tag)
- throw new RuntimeException("tag is not ordered correctly: "+prevTag+" > "+tag);
-
- remainingbuffer.set(secondIndex, aggregateValue);
- try {
- processJoin(tag, remainingbuffer);
- } catch (Exception e) {
- throw new IOException(e);
- }
- prevTag=tag;
-
- report.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis()-start);
- }
-
- private void processJoin(int tag, RemainIndexValue rValue)
- throws Exception
- {
-
- //for the cached matrix
- if(tag==0)
- {
- addToCache(rValue, tag);
- // LOG.info("put in the buffer for left matrix");
- // LOG.info(rblock.block.toString());
- }
- else//for the probing matrix
- {
- //LOG.info("process join with block size: "+rValue.value.getNumRows()+" X "+rValue.value.getNumColumns()+" nonZeros: "+rValue.value.getNonZeros());
- for(int i=0; i<cacheSize; i++)
- {
- RemainIndexValue left, right;
- if(tagForLeft==0)
- {
- left=cache.get(i);
- right=rValue;
- }else
- {
- right=cache.get(i);
- left=rValue;
- }
- indexesbuffer.setIndexes(left.remainIndex, right.remainIndex);
- try {
- OperationsOnMatrixValues.performAggregateBinaryIgnoreIndexes(left.value,
- right.value, valueBuffer, (AggregateBinaryOperator)aggBinInstruction.getOperator());
- } catch (DMLRuntimeException e) {
- throw new IOException(e);
- }
-
- //if(valueBuffer.getNonZeros()>0)
- collectOutput(indexesbuffer, valueBuffer);
- }
- }
- }
-
- private void collectOutput(MatrixIndexes indexes,
- MatrixValue value_out)
- throws Exception
- {
- MatrixValue value=outCache.get(indexes);
-
- try {
- if(value!=null)
- {
- //LOG.info("********** oops, should not run this code1 ***********");
-/* LOG.info("the output is in the cache");
- LOG.info("old block");
- LOG.info(block.toString());
-*/
- value.binaryOperationsInPlace(((AggregateBinaryOperator)aggBinInstruction.getOperator()).aggOp.increOp,
- value_out);
-
-/* LOG.info("add block");
- LOG.info(block_out.toString());
- LOG.info("result block");
- LOG.info(block.toString());
-*/
- }
- else if(outCache.size()<OUT_CACHE_SIZE)
- {
- //LOG.info("********** oops, should not run this code2 ***********");
- value=valueClass.newInstance();
- value.reset(value_out.getNumRows(), value_out.getNumColumns(), value.isInSparseFormat());
- value.binaryOperationsInPlace(((AggregateBinaryOperator)aggBinInstruction.getOperator()).aggOp.increOp,
- value_out);
- outCache.put(new MatrixIndexes(indexes), value);
-
-/* LOG.info("the output is not in the cache");
- LOG.info("result block");
- LOG.info(block.toString());
-*/
- }else
- {
- realWriteToCollector(indexes, value_out);
- }
- } catch (DMLRuntimeException e) {
- throw new IOException(e);
- }
- }
-
- private void resetCache() {
- cacheSize=0;
- }
-
- private void addToCache(RemainIndexValue rValue, int tag) throws Exception {
-
- //LOG.info("add to cache with block size: "+rValue.value.getNumRows()+" X "+rValue.value.getNumColumns()+" nonZeros: "+rValue.value.getNonZeros());
- if(cacheSize<cache.size())
- cache.get(cacheSize).set(rValue.remainIndex, rValue.value);
- else
- cache.add(new RemainIndexValue(rValue.remainIndex, rValue.value));
- cacheSize++;
- }
-
- //output the records in the outCache.
- public void close() throws IOException
- {
- long start=System.currentTimeMillis();
- Iterator<Entry<MatrixIndexes, MatrixValue>> it=outCache.entrySet().iterator();
- while(it.hasNext())
- {
- Entry<MatrixIndexes, MatrixValue> entry=it.next();
- realWriteToCollector(entry.getKey(), entry.getValue());
- }
-
- //handle empty block output (on first reduce task only)
- if( outputDummyRecords ) //required for rejecting empty blocks in mappers
- {
- long rlen = dim1.getRows();
- long clen = dim2.getCols();
- int brlen = dim1.getRowsPerBlock();
- int bclen = dim2.getColsPerBlock();
- MatrixIndexes tmpIx = new MatrixIndexes();
- MatrixBlock tmpVal = new MatrixBlock();
- for(long i=0, r=1; i<rlen; i+=brlen, r++)
- for(long j=0, c=1; j<clen; j+=bclen, c++)
- {
- int realBrlen=(int)Math.min((long)brlen, rlen-(r-1)*brlen);
- int realBclen=(int)Math.min((long)bclen, clen-(c-1)*bclen);
- tmpIx.setIndexes(r, c);
- tmpVal.reset(realBrlen,realBclen);
- collectFinalMultipleOutputs.collectOutput(tmpIx, tmpVal, 0, cachedReporter);
- }
- }
-
- if(cachedReporter!=null)
- cachedReporter.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis()-start);
- super.close();
- }
-
- public void realWriteToCollector(MatrixIndexes indexes, MatrixValue value) throws IOException
- {
- collectOutput_N_Increase_Counter(indexes, value, 0, cachedReporter);
-// LOG.info("--------- output: "+indexes+" <--> "+block);
-
- /* if(count%1000==0)
- {
- LOG.info("result block: sparse format="+value.isInSparseFormat()+
- ", dimension="+value.getNumRows()+"x"+value.getNumColumns()
- +", nonZeros="+value.getNonZeros());
- }
- count++;*/
-
- }
-
- public void configure(JobConf job)
- {
- super.configure(job);
- if(resultIndexes.length>1)
- throw new RuntimeException("MMCJMR only outputs one result");
-
- outputDummyRecords = MapReduceTool.getUniqueKeyPerTask(job, false).equals("0");
-
- try {
- //valueBuffer=valueClass.newInstance();
- valueBuffer=buffer;
- remainingbuffer=new RemainIndexValue(valueClass);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- int blockRlen=dim1.getRowsPerBlock();
- int blockClen=dim2.getColsPerBlock();
- int elementSize=(int)Math.ceil((double)(77+8*blockRlen*blockClen+20+12)/0.75);
- OUT_CACHE_SIZE=((long)OptimizerUtils.getLocalMemBudget() //current jvm max mem
- -MRJobConfiguration.getMMCJCacheSize(job))/elementSize;
- outCache=new HashMap<MatrixIndexes, MatrixValue>(1024);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
deleted file mode 100644
index e4cff5a..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactDoubleIntInputFormat.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.sort;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-
-import org.apache.sysml.runtime.matrix.data.MatrixCell;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.Pair;
-
-
-public class CompactDoubleIntInputFormat extends FileInputFormat<MatrixIndexes, MatrixCell>
-{
-
- public static final String SELECTED_RANGES_PREFIX="selected.ranges.in.";
- public static final String SELECTED_POINTS_PREFIX="selected.points.in.";
-
- public static void getPointsInEachPartFile(long[] counts, double[] probs, HashMap<Integer,
- ArrayList<Pair<Integer, Integer>>> posMap)
- {
- long[] ranges=new long[counts.length];
- ranges[0]=counts[0];
- for(int i=1; i<counts.length; i++)
- ranges[i]=ranges[i-1]+counts[i];
-
- long total=ranges[ranges.length-1];
-
- TreeMap<Double, Integer> sortedProbs=new TreeMap<Double, Integer>();
- for(int i=0; i<probs.length; i++)
- sortedProbs.put(probs[i], i);
-
- int currentPart=0;
- for(Entry<Double, Integer> e: sortedProbs.entrySet() )
- {
- long pos=(long)Math.ceil(total*e.getKey());
- while(ranges[currentPart]<pos)
- currentPart++;
- ArrayList<Pair<Integer, Integer>> vec=posMap.get(currentPart);
- if(vec==null)
- {
- vec=new ArrayList<Pair<Integer, Integer>>();
- posMap.put(currentPart, vec);
- }
- if(currentPart>0)
- vec.add( new Pair<Integer, Integer>( (int)(pos-ranges[currentPart-1]-1), e.getValue()));
- else
- vec.add( new Pair<Integer, Integer>( (int)pos-1, e.getValue()));
- }
- }
-
- public static Set<Integer> setRecordCountInEachPartFile(JobConf job, long[] counts, double[] probs)
- {
- HashMap<Integer, ArrayList<Pair<Integer, Integer>>> posMap
- =new HashMap<Integer, ArrayList<Pair<Integer, Integer>>>();
-
- getPointsInEachPartFile(counts, probs, posMap);
-
- for(Entry<Integer, ArrayList<Pair<Integer, Integer>>> e: posMap.entrySet())
- {
- job.set(SELECTED_POINTS_PREFIX+e.getKey(), getString(e.getValue()));
- }
-
- return posMap.keySet();
- }
-
- private static String getString(ArrayList<Pair<Integer, Integer>> value)
- {
- StringBuilder sb = new StringBuilder();
- for(Pair<Integer, Integer> i: value) {
- sb.append(i.getKey());
- sb.append(":");
- sb.append(i.getValue());
- sb.append(",");
- }
- return sb.substring(0, sb.length()-1);
- }
-
- public RecordReader<MatrixIndexes, MatrixCell> getRecordReader(InputSplit split
- , JobConf job, Reporter reporter) throws IOException {
- return new DoubleIntRecordReader(job, (FileSplit) split);
- }
-
- public static class DoubleIntRecordReader implements RecordReader<MatrixIndexes, MatrixCell>
- {
- protected long totLength;
- protected FileSystem fs;
- protected Path path;
- protected JobConf conf;
-
- protected FSDataInputStream currentStream;
- private int posIndex=0;
-
- private int[] pos; //starting from 0
- private int[] indexes;
- DoubleWritable readKey=new DoubleWritable();
- IntWritable readValue=new IntWritable();
- private int numRead=0;
- private boolean noRecordsNeeded=false;
-
- private int getIndexInTheArray(String name)
- {
- int i=name.indexOf("part-");
- assert(i>=0);
- return Integer.parseInt(name.substring(i+5));
- }
-
- public DoubleIntRecordReader(JobConf job, FileSplit split)
- throws IOException {
- fs = FileSystem.get(job);
- path = split.getPath();
- totLength = split.getLength();
- currentStream = fs.open(path);
- int partIndex=getIndexInTheArray(path.getName());
- String arrStr=job.get(SELECTED_POINTS_PREFIX+partIndex);
- if(arrStr==null || arrStr.isEmpty()) {
- noRecordsNeeded=true;
- return;
- }
-
- String[] strs=arrStr.split(",");
- pos=new int[strs.length];
- indexes=new int[strs.length];
- for(int i=0; i<strs.length; i++)
- {
- String[] temp=strs[i].split(":");
- pos[i]=Integer.parseInt(temp[0]);
- indexes[i]=Integer.parseInt(temp[1]);
- }
- }
-
- public boolean next(MatrixIndexes key, MatrixCell value)
- throws IOException {
-
- if(noRecordsNeeded || currentStream.getPos()>=totLength || posIndex>=pos.length)
- return false;
-
- while(numRead<=pos[posIndex])
- {
- readKey.readFields(currentStream);
- readValue.readFields(currentStream);
- numRead=readValue.get();
- }
-
- key.setIndexes(indexes[posIndex]+1, 1);
- value.setValue(readKey.get());
- posIndex++;
- //System.out.println("next: "+key+", "+value);
- return true;
- }
-
- @Override
- public void close() throws IOException {
- currentStream.close();
-
- }
-
- @Override
- public MatrixIndexes createKey() {
- return new MatrixIndexes();
- }
-
- @Override
- public MatrixCell createValue() {
- return new MatrixCell();
- }
-
- @Override
- public long getPos() throws IOException {
- long currentOffset = currentStream == null ? 0 : currentStream.getPos();
- return currentOffset;
- }
-
- @Override
- public float getProgress() throws IOException {
- return (float)posIndex/(float)pos.length;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/11381c12/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
index fc6b930..6a66111 100644
--- a/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/util/BinaryBlockInputFormat.java
@@ -33,8 +33,10 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Custom binary block input format to return the custom record reader.
- *
- * NOTE: not used by default.
+ * <p>
+ * NOTE: Not used by default.
+ * <p>
+ * NOTE: Used for performance debugging of binary block HDFS reads.
*/
public class BinaryBlockInputFormat extends SequenceFileInputFormat<MatrixIndexes,MatrixBlock>
{