You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2008/12/11 01:59:08 UTC
svn commit: r725509 - in /incubator/hama/trunk/src:
examples/org/apache/hama/examples/MatrixMultiplication.java
java/org/apache/hama/DenseMatrix.java
test/org/apache/hama/TestDenseMatrix.java
test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
Author: edwardyoon
Date: Wed Dec 10 16:59:05 2008
New Revision: 725509
URL: http://svn.apache.org/viewvc?rev=725509&view=rev
Log:
Add argument for the number of blocks
Modified:
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java Wed Dec 10 16:59:05 2008
@@ -27,7 +27,7 @@
public class MatrixMultiplication extends AbstractExample {
public static void main(String[] args) throws IOException {
if (args.length < 2) {
- System.out.println("mult [-m maps] [-r reduces] <matrix_A> <matrix_B>");
+ System.out.println("mult [-m maps] [-r reduces] <matrix_A> <matrix_B> [blocks]");
System.exit(-1);
} else {
parseArgs(args);
@@ -39,11 +39,11 @@
DenseMatrix a = new DenseMatrix(conf, matrixA, false);
DenseMatrix b = new DenseMatrix(conf, matrixB, false);
- if (!a.isBlocked())
- a.blocking_mapred(conf.getNumMapTasks());
- if (!b.isBlocked())
- b.blocking_mapred(conf.getNumMapTasks());
-
+ if(ARGS.size() > 2) {
+ a.blocking_mapred(Integer.parseInt(ARGS.get(2)));
+ b.blocking_mapred(Integer.parseInt(ARGS.get(2)));
+ }
+
Matrix c = a.mult(b);
c.close();
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Wed Dec 10 16:59:05 2008
@@ -464,28 +464,6 @@
return result;
}
- /**
- * Using a map/reduce job to block a dense matrix.
- *
- * @param blockNum
- * @throws IOException
- */
- public void blocking_mapred(int blockNum) throws IOException {
- setBlockPosition(blockNum);
- setBlockSize(blockNum);
- LOG.info("Convert to " + blockNum + " * " + blockNum + " blocked matrix");
-
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("Blocking MR job" + getPath());
-
- jobConf.setNumMapTasks(config.getNumMapTasks());
- jobConf.setNumReduceTasks(config.getNumReduceTasks());
-
- BlockingMapRed.initJob(getPath(), jobConf);
-
- JobManager.execute(jobConf);
- }
-
public boolean isBlocked() throws IOException {
return (table.get(Constants.METADATA, Constants.BLOCK_SIZE) == null) ? false
: true;
@@ -557,6 +535,26 @@
}
/**
+ * Using a map/reduce job to block a dense matrix.
+ *
+ * @param blockNum
+ * @throws IOException
+ */
+ public void blocking_mapred(int blockNum) throws IOException {
+ this.checkBlockNum(blockNum);
+
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("Blocking MR job" + getPath());
+
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+ BlockingMapRed.initJob(getPath(), jobConf);
+
+ JobManager.execute(jobConf);
+ }
+
+ /**
* Using a scanner to block a dense matrix. If the matrix is large, use the
* blocking_mapred()
*
@@ -564,9 +562,8 @@
* @throws IOException
*/
public void blocking(int blockNum) throws IOException {
- setBlockPosition(blockNum);
- setBlockSize(blockNum);
-
+ this.checkBlockNum(blockNum);
+
String[] columns = new String[] { Constants.BLOCK_STARTROW,
Constants.BLOCK_ENDROW, Constants.BLOCK_STARTCOLUMN,
Constants.BLOCK_ENDCOLUMN };
@@ -580,4 +577,16 @@
setBlock(blockR, blockC, subMatrix(pos[0], pos[1], pos[2], pos[3]));
}
}
+
+ private void checkBlockNum(int blockNum) throws IOException {
+ double blocks = Math.pow(blockNum, 0.5);
+ // TODO: Check also it is validation with matrix.
+ if(!String.valueOf(blocks).endsWith(".0"))
+ throw new IOException("can't divide.");
+
+ int block_size = (int) blocks;
+ setBlockPosition(block_size);
+ setBlockSize(block_size);
+ LOG.info("Create " + block_size + " * " + block_size + " blocked matrix");
+ }
}
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Wed Dec 10 16:59:05 2008
@@ -84,7 +84,7 @@
public void testBlocking() throws IOException, ClassNotFoundException {
assertEquals(((DenseMatrix) m1).isBlocked(), false);
- ((DenseMatrix) m1).blocking(2);
+ ((DenseMatrix) m1).blocking(4);
assertEquals(((DenseMatrix) m1).isBlocked(), true);
int[] pos = ((DenseMatrix) m1).getBlockPosition(1, 0);
double[][] b = ((DenseMatrix) m1).subMatrix(pos[0], pos[1], pos[2], pos[3]).getDoubleArray();
@@ -107,7 +107,7 @@
*/
public void testMRBlocking() throws IOException, ClassNotFoundException {
assertEquals(((DenseMatrix) m2).isBlocked(), false);
- ((DenseMatrix) m2).blocking_mapred(2);
+ ((DenseMatrix) m2).blocking_mapred(4);
assertEquals(((DenseMatrix) m2).isBlocked(), true);
int[] pos = ((DenseMatrix) m2).getBlockPosition(1, 0);
double[][] b = ((DenseMatrix) m2).subMatrix(pos[0], pos[1], pos[2], pos[3]).getDoubleArray();
Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java Wed Dec 10 16:59:05 2008
@@ -25,8 +25,8 @@
public void testBlockMatrixMapReduce() throws IOException, ClassNotFoundException {
Matrix m1 = DenseMatrix.random(conf, SIZE, SIZE);
Matrix m2 = DenseMatrix.random(conf, SIZE, SIZE);
- ((DenseMatrix) m1).blocking_mapred(2);
- ((DenseMatrix) m2).blocking_mapred(2);
+ ((DenseMatrix) m1).blocking_mapred(4);
+ ((DenseMatrix) m2).blocking_mapred(4);
miniMRJob(m1.getPath(), m2.getPath());