You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2008/09/14 10:56:17 UTC
svn commit: r695149 - in /incubator/hama/trunk: ./
src/examples/org/apache/hama/examples/ src/java/org/apache/hama/
src/java/org/apache/hama/mapred/ src/test/org/apache/hama/
Author: edwardyoon
Date: Sun Sep 14 01:56:17 2008
New Revision: 695149
URL: http://svn.apache.org/viewvc?rev=695149&view=rev
Log:
Write dimension attributes when job completed
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Sun Sep 14 01:56:17 2008
@@ -48,6 +48,7 @@
BUG FIXES
+ HAMA-55: Write dimension attributes when job completed (edwardyoon)
HAMA-54: Split doesn't split by map task num (edwardyoon)
HAMA-53: NullPointerException on distributed cluster (edwardyoon)
HAMA-26: hama-formatter.xml should be removed (edwardyoon)
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java Sun Sep 14 01:56:17 2008
@@ -43,16 +43,16 @@
System.out.println("\nMatrix A");
System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
+ for(int i = 0; i < a.getRows(); i++) {
+ for(int j = 0; j < a.getColumns(); j++) {
System.out.println(a.get(i, j));
}
}
System.out.println("\nMatrix B");
System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
+ for(int i = 0; i < b.getRows(); i++) {
+ for(int j = 0; j < b.getColumns(); j++) {
System.out.println(b.get(i, j));
}
}
@@ -62,8 +62,8 @@
System.out.println("\nC = A + B");
System.out.println("----------------------");
- for(int i = 0; i < row; i++) {
- for(int j = 0; j < row; j++) {
+ for(int i = 0; i < c.getRows(); i++) {
+ for(int j = 0; j < c.getColumns(); j++) {
System.out.println(c.get(i, j));
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Sun Sep 14 01:56:17 2008
@@ -28,6 +28,9 @@
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
import org.apache.hama.util.Numeric;
import org.apache.log4j.Logger;
@@ -75,6 +78,17 @@
}
}
+ public void execute(JobConf jobConf, Matrix result) throws IOException {
+ RunningJob rJob = JobClient.runJob(jobConf);
+ // TODO : When HADOOP-4043 done, we should change this.
+ long rows = rJob.getCounters().findCounter(
+ "org.apache.hadoop.mapred.Task$Counter", 8, "REDUCE_OUTPUT_RECORDS")
+ .getCounter();
+ // TODO : Thinking about more efficient method.
+ int columns = result.getRow(0).size();
+ result.setDimension((int)rows, columns);
+ }
+
/** {@inheritDoc} */
public double get(int i, int j) throws IOException {
double result = -1;
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Sun Sep 14 01:56:17 2008
@@ -28,7 +28,6 @@
import org.apache.hadoop.hbase.client.Scanner;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hama.algebra.AdditionMap;
import org.apache.hama.algebra.AdditionReduce;
@@ -133,18 +132,17 @@
public Matrix add(Matrix B) throws IOException {
String output = RandomVariable.randMatrixName();
- Matrix C = new DenseMatrix(config, output);
+ Matrix result = new DenseMatrix(config, output);
JobConf jobConf = new JobConf(config);
- jobConf.setJobName("addition MR job");
- jobConf.setNumMapTasks(2);
+ jobConf.setJobName("addition MR job" + result.getName());
AdditionMap.initJob(this.getName(), B.getName(), AdditionMap.class,
IntWritable.class, DenseVector.class, jobConf);
- MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
+ MatrixReduce.initJob(result.getName(), AdditionReduce.class, jobConf);
- JobClient.runJob(jobConf);
- return C;
+ execute(jobConf, result);
+ return result;
}
public Matrix add(double alpha, Matrix B) throws IOException {
@@ -173,17 +171,16 @@
public Matrix mult(Matrix B) throws IOException {
String output = RandomVariable.randMatrixName();
- Matrix C = new DenseMatrix(config, output);
+ Matrix result = new DenseMatrix(config, output);
JobConf jobConf = new JobConf(config);
- jobConf.setJobName("multiplication MR job");
+ jobConf.setJobName("multiplication MR job : " + result.getName());
- MultiplicationMap.initJob(this.getName(), B.getName(), MultiplicationMap.class,
- IntWritable.class, DenseVector.class, jobConf);
- MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
-
- JobClient.runJob(jobConf);
- return C;
+ MultiplicationMap.initJob(this.getName(), B.getName(),
+ MultiplicationMap.class, IntWritable.class, DenseVector.class, jobConf);
+ MatrixReduce.initJob(result.getName(), MultiplicationReduce.class, jobConf);
+ execute(jobConf, result);
+ return result;
}
public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException {
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java Sun Sep 14 01:56:17 2008
@@ -37,11 +37,8 @@
/**
* space delimited list of columns
- *
- * @see org.apache.hadoop.hbase.regionserver.HAbstractScanner for column name
- * wildcards
*/
- public static final String COLUMN_LIST = "hbase.mapred.tablecolumns";
+ public static final String COLUMN_LIST = "hama.mapred.tablecolumns";
/** {@inheritDoc} */
public void configure(JobConf job) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java Sun Sep 14 01:56:17 2008
@@ -32,7 +32,6 @@
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.mapred.TableSplit;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.InputFormat;
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java Sun Sep 14 01:56:17 2008
@@ -27,7 +27,6 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.mapred.TableOutputFormat;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -41,8 +40,8 @@
FileOutputFormat<IntWritable, BatchUpdate> {
/** JobConf parameter that specifies the output table */
- public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
- private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
+ public static final String OUTPUT_TABLE = "hama.mapred.output";
+ private final static Log LOG = LogFactory.getLog(MatrixOutputFormat.class);
/**
* Convert Reduce output (key, value) to (HStoreKey, KeyedDataArrayWritable)
@@ -63,7 +62,6 @@
/** {@inheritDoc} */
public void close(Reporter reporter) {
- // TODO : Sets the (row, column) dimension
}
/** {@inheritDoc} */
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Sun Sep 14 01:56:17 2008
@@ -35,7 +35,7 @@
*/
public class TestDenseMatrix extends TestCase {
static final Logger LOG = Logger.getLogger(TestDenseMatrix.class);
- private static int SIZE = 5;
+ private static int SIZE = 10;
private static Matrix m1;
private static Matrix m2;
@@ -80,6 +80,9 @@
public void testMatrixAdd() throws IOException {
Matrix result = m1.add(m2);
+ assertEquals(result.getRows(), SIZE);
+ assertEquals(result.getColumns(), SIZE);
+
for (int i = 0; i < SIZE; i++) {
for (int j = 0; j < SIZE; j++) {
assertEquals(result.get(i, j), m1.get(i, j) + m2.get(i, j));
@@ -95,31 +98,36 @@
public void testMatrixMult() throws IOException {
Matrix result = m1.mult(m2);
- verifyMultResult(SIZE, m1, m2, result);
+ assertEquals(result.getRows(), SIZE);
+ assertEquals(result.getColumns(), SIZE);
+
+ verifyMultResult(m1, m2, result);
}
/**
* Verifying multiplication result
*
- * @param size
* @param m1
* @param m2
* @param result
* @throws IOException
*/
- private void verifyMultResult(int size, Matrix m1, Matrix m2, Matrix result)
+ private void verifyMultResult(Matrix m1, Matrix m2, Matrix result)
throws IOException {
double[][] C = new double[SIZE][SIZE];
- for (int i = 0; i < SIZE; i++)
- for (int j = 0; j < SIZE; j++)
- for (int k = 0; k < SIZE; k++)
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
+ for (int k = 0; k < SIZE; k++) {
C[i][k] += m1.get(i, j) * m2.get(j, k);
+ }
+ }
+ }
- for (int i = 0; i < 2; i++) {
- for (int j = 0; j < 2; j++) {
- LOG.info("result: " + result.get(i, j) + ", C: " + C[i][j]);
- assertEquals(result.get(i, j), C[i][j]);
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
+ assertEquals(String.valueOf(result.get(i, j)).substring(0, 14),
+ String.valueOf(C[i][j]).substring(0, 14));
}
}
}