You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/06/04 10:53:00 UTC
svn commit: r781687 - in /incubator/hama/trunk: ./
src/examples/org/apache/hama/examples/ src/java/org/apache/hama/
src/java/org/apache/hama/mapred/ src/java/org/apache/hama/util/
src/test/org/apache/hama/mapred/ src/test/org/apache/hama/util/
Author: edwardyoon
Date: Thu Jun 4 08:53:00 2009
New Revision: 781687
URL: http://svn.apache.org/viewvc?rev=781687&view=rev
Log:
Add density option for random matrix
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java
incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java
incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Thu Jun 4 08:53:00 2009
@@ -38,6 +38,7 @@
IMPROVEMENTS
+ HAMA-186: Add density option for random matrix (edwardyoon)
HAMA-171: Implementation of Matrix.Norm (samuel)
HAMA-178: Add example of norm (edwardyoon)
HAMA-158: Implementation of random sparse matrix (edwardyoon)
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java Thu Jun 4 08:53:00 2009
@@ -30,7 +30,9 @@
public static void main(String[] args) throws IOException {
if (args.length < 3) {
System.out
- .println("rand [-m maps] [-r reduces] <rows> <columns> <sparse | dense> <matrix_name>");
+ .println("rand [-m maps] [-r reduces] <rows> <columns> <density> <matrix_name>");
+ System.out
+ .println("ex) rand -m 10 -r 10 2000 2000 30% matrixA");
System.exit(-1);
} else {
parseArgs(args);
@@ -38,12 +40,13 @@
int row = Integer.parseInt(ARGS.get(0));
int column = Integer.parseInt(ARGS.get(1));
-
+ int percent = Integer.parseInt(ARGS.get(2).substring(0, ARGS.get(2).length()-1));
+
Matrix a;
- if(ARGS.get(2).equals("sparse"))
- a = SparseMatrix.random_mapred(conf, row, column);
- else
+ if(percent == 100)
a = DenseMatrix.random_mapred(conf, row, column);
+ else
+ a = SparseMatrix.random_mapred(conf, row, column, percent);
a.save(ARGS.get(3));
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Thu Jun 4 08:53:00 2009
@@ -243,9 +243,10 @@
RandomMatrixReduce.initJob(rand.getPath(), RandomMatrixReduce.class,
jobConf);
jobConf.setSpeculativeExecution(false);
- jobConf.set("matrix.column", String.valueOf(n));
+ jobConf.setInt("matrix.column", n);
jobConf.set("matrix.type", TABLE_PREFIX);
-
+ jobConf.setInt("matrix.density", 100);
+
jobConf.setInputFormat(SequenceFileInputFormat.class);
final FileSystem fs = FileSystem.get(jobConf);
int interval = m / conf.getNumMapTasks();
Modified: incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java Thu Jun 4 08:53:00 2009
@@ -109,7 +109,7 @@
return rand;
}
- public static SparseMatrix random_mapred(HamaConfiguration conf, int m, int n) throws IOException {
+ public static SparseMatrix random_mapred(HamaConfiguration conf, int m, int n, int percent) throws IOException {
SparseMatrix rand = new SparseMatrix(conf, m, n);
LOG.info("Create the " + m + " * " + n + " random matrix : "
+ rand.getPath());
@@ -129,8 +129,9 @@
RandomMatrixReduce.initJob(rand.getPath(), RandomMatrixReduce.class,
jobConf);
jobConf.setSpeculativeExecution(false);
- jobConf.set("matrix.column", String.valueOf(n));
+ jobConf.setInt("matrix.column", n);
jobConf.set("matrix.type", TABLE_PREFIX);
+ jobConf.setInt("matrix.density", percent);
jobConf.setInputFormat(SequenceFileInputFormat.class);
final FileSystem fs = FileSystem.get(jobConf);
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java Thu Jun 4 08:53:00 2009
@@ -20,7 +20,6 @@
package org.apache.hama.mapred;
import java.io.IOException;
-import java.util.Random;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.MapWritable;
@@ -41,7 +40,7 @@
public class RandomMatrixMap extends MapReduceBase implements
Mapper<IntWritable, IntWritable, IntWritable, MapWritable> {
static final Logger LOG = Logger.getLogger(RandomMatrixMap.class);
- protected int column;
+ protected int column, density;
protected String type;
protected Vector vector = new DenseVector();
@@ -53,9 +52,7 @@
((SparseVector) vector).clear();
for (int i = key.get(); i <= value.get(); i++) {
for (int j = 0; j < column; j++) {
- Random r = new Random();
- if(r.nextInt(2) != 0)
- ((SparseVector) vector).set(j, RandomVariable.rand());
+ ((SparseVector) vector).set(j, RandomVariable.rand(density));
}
output.collect(new IntWritable(i), vector.getEntries());
}
@@ -71,7 +68,8 @@
}
public void configure(JobConf job) {
- column = Integer.parseInt(job.get("matrix.column"));
+ column = job.getInt("matrix.column", 0);
+ density = job.getInt("matrix.density", 100);
type = job.get("matrix.type");
if (type.equals("SparseMatrix"))
vector = new SparseVector();
Modified: incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java Thu Jun 4 08:53:00 2009
@@ -26,7 +26,7 @@
* numbers.
*/
public class RandomVariable {
-
+
/**
* Generate a random number between 0 and 1.
*
@@ -38,6 +38,23 @@
}
/**
+ * Generate a random number
+ *
+ * @param density percentage of the non-zero
+ * @return a double 0 or between 0 and 1
+ */
+ public static double rand(int density) {
+ int[] prob = new int[100];
+ double x = Math.random();
+
+ for(int i = 0; i < density; i++) {
+ prob[i] = 1;
+ }
+
+ return (prob[randInt(0, 99)] == 0) ? 0 : x;
+ }
+
+ /**
* Generate a random integer.
*
* @param i0 min of the random variable.
Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java Thu Jun 4 08:53:00 2009
@@ -42,7 +42,7 @@
rand.close();
- SparseMatrix rand2 = SparseMatrix.random_mapred(conf, 20, 20);
+ SparseMatrix rand2 = SparseMatrix.random_mapred(conf, 20, 20, 30);
assertEquals(20, rand2.getRows());
assertEquals(20, rand2.getColumns());
boolean zeroAppear = false;
Modified: incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java Thu Jun 4 08:53:00 2009
@@ -19,12 +19,15 @@
*/
package org.apache.hama.util;
+import org.apache.log4j.Logger;
+
import junit.framework.TestCase;
/**
* Random variable generation test
*/
public class TestRandomVariable extends TestCase {
+ static final Logger LOG = Logger.getLogger(TestRandomVariable.class);
final static int COUNT = 50;
/**
@@ -37,6 +40,13 @@
double result = RandomVariable.rand();
assertTrue(result >= 0.0d && result <= 1.0);
}
+
+ int nonZero = 0;
+ for (int i = 0; i < COUNT; i++) {
+ if(RandomVariable.rand(70) > 0)
+ nonZero++;
+ }
+ assertTrue((COUNT/2) < nonZero);
}
/**