You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/10/27 07:04:12 UTC
svn commit: r830067 - in /incubator/hama/trunk: ./
src/examples/org/apache/hama/examples/ src/java/org/apache/hama/matrix/
src/java/org/apache/hama/matrix/algebra/ src/test/org/apache/hama/examples/
src/test/org/apache/hama/matrix/
Author: edwardyoon
Date: Tue Oct 27 06:04:05 2009
New Revision: 830067
URL: http://svn.apache.org/viewvc?rev=830067&view=rev
Log:
Replacement of Mat-Mat addition Map/Reduce
Added:
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionMap.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionReduce.java
Removed:
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixNormMap.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixNormReduce.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/RowCyclicAdditionMap.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/RowCyclicAdditionReduce.java
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/DenseMatrixVectorMultMap.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/SparseMatrixVectorMultMap.java
incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/matrix/TestAbstractMatrix.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Oct 27 06:04:05 2009
@@ -42,6 +42,7 @@
IMPROVEMENTS
+ HAMA-207: Replacement of Mat-Mat addition Map/Reduce (edwardyoon)
HAMA-208: Replacement of vector-matrix multiplication Map/Reduce (edwardyoon)
HAMA-216: Removing JobManager in util package (edwardyoon)
HAMA-215: Removing hama shell from version 0.2 (edwardyoon)
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java Tue Oct 27 06:04:05 2009
@@ -65,7 +65,7 @@
public void setConf(Configuration conf) {
this.conf = conf;
try {
- matrix = new DenseMatrix(new HamaConfiguration(), conf
+ matrix = new DenseMatrix(new HamaConfiguration(conf), conf
.get("input.matrix"));
} catch (IOException e) {
e.printStackTrace();
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java Tue Oct 27 06:04:05 2009
@@ -70,8 +70,8 @@
import org.apache.hama.matrix.algebra.DenseMatrixVectorMultMap;
import org.apache.hama.matrix.algebra.DenseMatrixVectorMultReduce;
import org.apache.hama.matrix.algebra.JacobiEigenValue;
-import org.apache.hama.matrix.algebra.RowCyclicAdditionMap;
-import org.apache.hama.matrix.algebra.RowCyclicAdditionReduce;
+import org.apache.hama.matrix.algebra.MatrixAdditionMap;
+import org.apache.hama.matrix.algebra.MatrixAdditionReduce;
import org.apache.hama.util.BytesUtil;
import org.apache.hama.util.RandomVariable;
@@ -446,18 +446,27 @@
DenseMatrix result = new DenseMatrix(config, this.getRows(), this
.getColumns());
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("addition MR job" + result.getPath());
-
- jobConf.setNumMapTasks(config.getNumMapTasks());
- jobConf.setNumReduceTasks(config.getNumReduceTasks());
+ Job job = new Job(config, "addition MR job" + result.getPath());
- RowCyclicAdditionMap.initJob(this.getPath(), B.getPath(), Double
- .toString(alpha), RowCyclicAdditionMap.class, IntWritable.class,
- MapWritable.class, jobConf);
- RowCyclicAdditionReduce.initJob(result.getPath(),
- RowCyclicAdditionReduce.class, jobConf);
- JobClient.runJob(jobConf);
+ Scan scan = new Scan();
+ scan.addFamily(Constants.COLUMNFAMILY);
+ job.getConfiguration().set(MatrixAdditionMap.MATRIX_SUMMANDS, B.getPath());
+ job.getConfiguration().set(MatrixAdditionMap.MATRIX_ALPHAS, Double
+ .toString(alpha));
+
+ TableMapReduceUtil.initTableMapperJob(this.getPath(), scan,
+ MatrixAdditionMap.class, IntWritable.class,
+ MapWritable.class, job);
+ TableMapReduceUtil.initTableReducerJob(result.getPath(),
+ MatrixAdditionReduce.class, job);
+ try {
+ job.waitForCompletion(true);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+
return result;
}
@@ -480,11 +489,6 @@
DenseMatrix result = new DenseMatrix(config, this.getRows(), this
.getColumns());
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("addition MR job" + result.getPath());
-
- jobConf.setNumMapTasks(config.getNumMapTasks());
- jobConf.setNumReduceTasks(config.getNumReduceTasks());
StringBuilder summandList = new StringBuilder();
StringBuilder alphaList = new StringBuilder();
@@ -497,13 +501,26 @@
summandList.deleteCharAt(summandList.length() - 1);
alphaList.deleteCharAt(alphaList.length() - 1);
- RowCyclicAdditionMap.initJob(this.getPath(), summandList.toString(),
- alphaList.toString(), RowCyclicAdditionMap.class, IntWritable.class,
- MapWritable.class, jobConf);
- RowCyclicAdditionReduce.initJob(result.getPath(),
- RowCyclicAdditionReduce.class, jobConf);
+ Job job = new Job(config, "addition MR job" + result.getPath());
- JobClient.runJob(jobConf);
+ Scan scan = new Scan();
+ scan.addFamily(Constants.COLUMNFAMILY);
+ job.getConfiguration().set(MatrixAdditionMap.MATRIX_SUMMANDS, summandList.toString());
+ job.getConfiguration().set(MatrixAdditionMap.MATRIX_ALPHAS, alphaList.toString());
+
+ TableMapReduceUtil.initTableMapperJob(this.getPath(), scan,
+ MatrixAdditionMap.class, IntWritable.class,
+ MapWritable.class, job);
+ TableMapReduceUtil.initTableReducerJob(result.getPath(),
+ MatrixAdditionReduce.class, job);
+ try {
+ job.waitForCompletion(true);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+
return result;
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/DenseMatrixVectorMultMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/DenseMatrixVectorMultMap.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/DenseMatrixVectorMultMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/DenseMatrixVectorMultMap.java Tue Oct 27 06:04:05 2009
@@ -60,7 +60,7 @@
this.conf = conf;
DenseMatrix matrix_a;
try {
- matrix_a = new DenseMatrix(new HamaConfiguration(), conf.get(MATRIX_A,
+ matrix_a = new DenseMatrix(new HamaConfiguration(conf), conf.get(MATRIX_A,
""));
int ithRow = conf.getInt(ITH_ROW, 0);
nKey.set(ithRow);
Added: incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionMap.java?rev=830067&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionMap.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionMap.java Tue Oct 27 06:04:05 2009
@@ -0,0 +1,65 @@
+package org.apache.hama.matrix.algebra;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.matrix.DenseMatrix;
+import org.apache.hama.matrix.DenseVector;
+import org.apache.hama.util.BytesUtil;
+import org.apache.log4j.Logger;
+
+public class MatrixAdditionMap extends TableMapper<IntWritable, MapWritable>
+ implements Configurable {
+ static final Logger LOG = Logger.getLogger(MatrixAdditionMap.class);
+ private Configuration conf = null;
+ protected DenseMatrix[] matrix_summands;
+ protected double[] matrix_alphas;
+ public static final String MATRIX_SUMMANDS = "hama.addition.summands";
+ public static final String MATRIX_ALPHAS = "hama.addition.alphas";
+
+ public void map(ImmutableBytesWritable key, Result value, Context context)
+ throws IOException, InterruptedException {
+ IntWritable nKey = new IntWritable(BytesUtil.getRowIndex(key.get()));
+
+ DenseVector result = new DenseVector(value);
+ DenseVector summand;
+ for (int i = 0; i < matrix_summands.length; i++) {
+ summand = matrix_summands[i].getRow(nKey.get());
+ result = result.add(matrix_alphas[i], summand);
+ }
+ context.write(nKey, result.getEntries());
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ try {
+ String[] matrix_names = conf.get(MATRIX_SUMMANDS, "").split(",");
+ String[] matrix_alpha_strs = conf.get(MATRIX_ALPHAS, "").split(",");
+ assert (matrix_names.length == matrix_alpha_strs.length && matrix_names.length >= 1);
+
+ matrix_summands = new DenseMatrix[matrix_names.length];
+ matrix_alphas = new double[matrix_names.length];
+ for (int i = 0; i < matrix_names.length; i++) {
+ matrix_summands[i] = new DenseMatrix(new HamaConfiguration(conf),
+ matrix_names[i]);
+ matrix_alphas[i] = Double.valueOf(matrix_alpha_strs[i]);
+ }
+ } catch (IOException e) {
+ LOG.warn("Load matrix_b failed : " + e.getMessage());
+ }
+ }
+
+}
Added: incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionReduce.java?rev=830067&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionReduce.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/MatrixAdditionReduce.java Tue Oct 27 06:04:05 2009
@@ -0,0 +1,28 @@
+package org.apache.hama.matrix.algebra;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableReducer;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.io.VectorUpdate;
+import org.apache.hama.util.BytesUtil;
+
+public class MatrixAdditionReduce extends
+TableReducer<IntWritable, MapWritable, Writable> {
+
+ @Override
+ public void reduce(IntWritable key, Iterable<MapWritable> values,
+ Context context) throws IOException, InterruptedException {
+
+ VectorUpdate update = new VectorUpdate(key.get());
+ for (MapWritable value : values) {
+ update.putAll(value);
+ }
+
+ context.write(new ImmutableBytesWritable(BytesUtil.getRowIndex(key.get())),
+ update.getPut());
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/SparseMatrixVectorMultMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/SparseMatrixVectorMultMap.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/SparseMatrixVectorMultMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/SparseMatrixVectorMultMap.java Tue Oct 27 06:04:05 2009
@@ -60,7 +60,7 @@
this.conf = conf;
SparseMatrix matrix_a;
try {
- matrix_a = new SparseMatrix(new HamaConfiguration(), conf.get(MATRIX_A,
+ matrix_a = new SparseMatrix(new HamaConfiguration(conf), conf.get(MATRIX_A,
""));
int ithRow = conf.getInt(ITH_ROW, 0);
nKey.set(ithRow);
Modified: incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java Tue Oct 27 06:04:05 2009
@@ -103,7 +103,7 @@
public void setConf(Configuration conf) {
this.conf = conf;
try {
- matrix = new DenseMatrix(new HamaConfiguration(), conf
+ matrix = new DenseMatrix(new HamaConfiguration(conf), conf
.get("input.matrix"));
} catch (IOException e) {
e.printStackTrace();
Modified: incubator/hama/trunk/src/test/org/apache/hama/matrix/TestAbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/matrix/TestAbstractMatrix.java?rev=830067&r1=830066&r2=830067&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/matrix/TestAbstractMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/matrix/TestAbstractMatrix.java Tue Oct 27 06:04:05 2009
@@ -51,7 +51,6 @@
double norm1 = matrix.norm(Norm.One);
double verify_norm1 = MatrixTestCommon.verifyNorm1(matrix);
gap = norm1 - verify_norm1;
- LOG.info(">>>>>> " + gap);
assertTrue(gap < 0.000001 && gap > -0.000001);
double normInfinity = matrix.norm(Norm.Infinity);