You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by sa...@apache.org on 2009/01/15 16:39:17 UTC
svn commit: r734736 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/
src/java/org/apache/hama/algebra/ src/java/org/apache/hama/mapred/
Author: samuel
Date: Thu Jan 15 07:39:16 2009
New Revision: 734736
URL: http://svn.apache.org/viewvc?rev=734736&view=rev
Log:
HAMA-144: GetProgress during MR over a matrix
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=734736&r1=734735&r2=734736&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Thu Jan 15 07:39:16 2009
@@ -33,6 +33,7 @@
IMPROVEMENTS
+ HAMA-144: GetProgress during MR over a matrix (samuel)
HAMA-129: Improving speed of matrix multiplication (edwardyoon)
HAMA-142: Trunk doesn't work for large matrices (edwardyoon)
HAMA-143: Improve of random_mapred() (edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java?rev=734736&r1=734735&r2=734736&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java Thu Jan 15 07:39:16 2009
@@ -210,14 +210,15 @@
}
public String toString() {
- String result = "";
+ StringBuilder result = new StringBuilder();
for (int i = 0; i < this.getRows(); i++) {
for (int j = 0; j < this.getColumns(); j++) {
- result += this.get(i, j) + "\t";
+ result.append(this.get(i, j));
+ result.append('\t');
}
- result += "\n";
+ result.append('\n');
}
- return result;
+ return result.toString();
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java?rev=734736&r1=734735&r2=734736&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java Thu Jan 15 07:39:16 2009
@@ -43,7 +43,7 @@
public void configure(JobConf job) {
try {
- matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+ matrix_b = new DenseMatrix(new HamaConfiguration(job), job.get(MATRIX_B, ""));
} catch (IOException e) {
LOG.warn("Load matrix_b failed : " + e.getMessage());
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=734736&r1=734735&r2=734736&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Thu Jan 15 07:39:16 2009
@@ -49,7 +49,7 @@
public void configure(JobConf job) {
try {
- matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+ matrix_b = new DenseMatrix(new HamaConfiguration(job), job.get(MATRIX_B, ""));
} catch (IOException e) {
LOG.warn("Load matrix_b failed : " + e.getMessage());
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java?rev=734736&r1=734735&r2=734736&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java Thu Jan 15 07:39:16 2009
@@ -26,6 +26,7 @@
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.mapred.TableSplit;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.InputFormat;
@@ -49,6 +50,27 @@
protected static class TableRecordReader extends TableRecordReaderBase
implements RecordReader<IntWritable, VectorWritable> {
+ private int totalRows;
+ private int processedRows;
+
+ @Override
+ public void init() throws IOException {
+ super.init();
+ if(endRow.length == 0) { // the last split, we don't know the end row
+ totalRows = 0; // so we just skip it.
+ } else {
+ if(startRow.length == 0) { // the first split, start row is 0
+ totalRows = BytesUtil.bytesToInt(endRow);
+ } else {
+ totalRows = BytesUtil.bytesToInt(endRow) - BytesUtil.bytesToInt(startRow);
+ }
+ }
+ processedRows = 0;
+ LOG.info("Split (" + Bytes.toString(startRow) + ", " + Bytes.toString(endRow) +
+ ") -> " + totalRows);
+ }
+
+
/**
* @return IntWritable
*
@@ -94,9 +116,20 @@
key.set(BytesUtil.bytesToInt(row));
lastRow = row;
Writables.copyWritable(result, value);
+ processedRows++;
}
return hasMore;
}
+
+ @Override
+ public float getProgress() {
+ if(totalRows <= 0) {
+ return 0;
+ } else {
+ return Math.min(1.0f, processedRows / (float)totalRows);
+ }
+ }
+
}
/**