You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2008/11/11 04:56:26 UTC
svn commit: r712931 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/
src/java/org/apache/hama/algebra/ src/java/org/apache/hama/io/
Author: edwardyoon
Date: Mon Nov 10 19:56:26 2008
New Revision: 712931
URL: http://svn.apache.org/viewvc?rev=712931&view=rev
Log:
Add getNumMap/reduceTasks to HamaConfiguration
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Nov 10 19:56:26 2008
@@ -3,7 +3,7 @@
Trunk (unreleased changes)
NEW FEATURES
-
+ HAMA-104: Add getNumMap/reduceTasks to HamaConfiguration (edwardyoon)
HAMA-92: Add subMatrix() to Matrix (edwardyoon)
HAMA-83: Add a writable comparable for BlockID (edwardyoon)
HAMA-81: Add subVector(int i0, int i1) to Vector (edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java Mon Nov 10 19:56:26 2008
@@ -32,7 +32,12 @@
}
public void set(int index, double value) {
- entries.put(index, new VectorEntry(value));
+ // If entries are null, create new object
+ if(this.entries == null) {
+ this.entries = new VectorMapWritable<Integer, VectorEntry>();
+ }
+
+ this.entries.put(index, new VectorEntry(value));
}
public void add(int index, double value) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Mon Nov 10 19:56:26 2008
@@ -204,8 +204,11 @@
public static Matrix random(HamaConfiguration conf, int m, int n)
throws IOException {
Matrix rand = new DenseMatrix(conf);
+ DenseVector vector = new DenseVector();
+ LOG.info("Create the " + m + " * " + n + " random matrix : " + rand.getPath());
+
for (int i = 0; i < m; i++) {
- DenseVector vector = new DenseVector();
+ vector.clear();
for (int j = 0; j < n; j++) {
vector.set(j, RandomVariable.rand());
}
@@ -213,7 +216,6 @@
}
rand.setDimension(m, n);
- LOG.info("Create the " + m + " * " + n + " random matrix : " + rand.getPath());
return rand;
}
@@ -229,7 +231,8 @@
public static Matrix identity(HamaConfiguration conf, int m, int n)
throws IOException {
Matrix identity = new DenseMatrix(conf);
-
+ LOG.info("Create the " + m + " * " + n + " identity matrix : " + identity.getPath());
+
for (int i = 0; i < m; i++) {
DenseVector vector = new DenseVector();
for (int j = 0; j < n; j++) {
@@ -239,7 +242,6 @@
}
identity.setDimension(m, n);
- LOG.info("Create the " + m + " * " + n + " identity matrix : " + identity.getPath());
return identity;
}
@@ -249,9 +251,8 @@
JobConf jobConf = new JobConf(config);
jobConf.setJobName("addition MR job" + result.getPath());
- jobConf.setNumMapTasks(Integer.parseInt(config.get("mapred.map.tasks")));
- jobConf.setNumReduceTasks(Integer.parseInt(config
- .get("mapred.reduce.tasks")));
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
RowCyclicAdditionMap.initJob(this.getPath(), B.getPath(), RowCyclicAdditionMap.class,
IntWritable.class, VectorWritable.class, jobConf);
@@ -267,15 +268,7 @@
}
public DenseVector getRow(int row) throws IOException {
- VectorMapWritable<Integer, VectorEntry> values = new VectorMapWritable<Integer, VectorEntry>();
- RowResult rowResult = table.getRow(String.valueOf(row));
-
- for (Map.Entry<byte[], Cell> f : rowResult.entrySet()) {
- VectorEntry entry = new VectorEntry(f.getValue());
- values.put(Numeric.getColumnIndex(f.getKey()), entry);
- }
-
- return new DenseVector(values);
+ return new DenseVector(table.getRow(Numeric.intToBytes(row)));
}
public Vector getColumn(int column) throws IOException {
@@ -286,8 +279,8 @@
VectorMapWritable<Integer, VectorEntry> trunk = new VectorMapWritable<Integer, VectorEntry>();
for (RowResult row : scan) {
- trunk.put(Numeric.bytesToInt(row.getRow()), new VectorEntry(row
- .get(columnKey)));
+ trunk.put(Numeric.bytesToInt(row.getRow()),
+ new VectorEntry(row.get(columnKey)));
}
return new DenseVector(trunk);
@@ -299,9 +292,8 @@
JobConf jobConf = new JobConf(config);
jobConf.setJobName("multiplication MR job : " + result.getPath());
- jobConf.setNumMapTasks(Integer.parseInt(config.get("mapred.map.tasks")));
- jobConf.setNumReduceTasks(Integer.parseInt(config
- .get("mapred.reduce.tasks")));
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), SIMDMultiplyMap.class,
IntWritable.class, VectorWritable.class, jobConf);
@@ -352,8 +344,7 @@
c[i] = Numeric.getColumnIndex(j0 + i);
}
- Scanner scan = table.getScanner(c, Numeric.intToBytes(i0), Numeric
- .intToBytes(i1 + 1));
+ Scanner scan = table.getScanner(c, Numeric.intToBytes(i0), Numeric.intToBytes(i1 + 1));
int rKey = 0, cKey = 0;
for (RowResult row : scan) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Mon Nov 10 19:56:26 2008
@@ -20,10 +20,14 @@
package org.apache.hama;
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hama.io.VectorEntry;
import org.apache.hama.io.VectorMapWritable;
+import org.apache.hama.util.Numeric;
import org.apache.log4j.Logger;
public class DenseVector extends AbstractVector implements Vector {
@@ -37,6 +41,14 @@
this.entries = m;
}
+ public DenseVector(RowResult row) {
+ this.entries = new VectorMapWritable<Integer, VectorEntry>();
+ for (Map.Entry<byte[], Cell> f : row.entrySet()) {
+ this.entries.put(Numeric.getColumnIndex(f.getKey()),
+ new VectorEntry(f.getValue()));
+ }
+ }
+
public Vector add(double alpha, Vector v) {
if (alpha == 0)
return this;
@@ -75,19 +87,9 @@
}
public Vector scale(double alpha) {
- Set<Integer> keySet = this.entries.keySet();
- Iterator<Integer> it = keySet.iterator();
-
- int i = 0;
- while (it.hasNext()) {
- int key = it.next();
- double oValue = this.get(key);
- double nValue = oValue * alpha;
-
- this.entries.put(i, new VectorEntry(nValue));
- i++;
+ for(Map.Entry<Integer, VectorEntry> e : this.entries.entrySet()) {
+ this.entries.put(e.getKey(), new VectorEntry(e.getValue().getValue() * alpha));
}
-
return this;
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java Mon Nov 10 19:56:26 2008
@@ -54,6 +54,14 @@
set("mapred.reduce.tasks",String.valueOf(reduce));
}
+ public int getNumMapTasks() {
+ return Integer.parseInt(get("mapred.map.tasks"));
+ }
+
+ public int getNumReduceTasks() {
+ return Integer.parseInt(get("mapred.reduce.tasks"));
+ }
+
private void addHbaseResources() {
addResource("hama-site.xml");
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java Mon Nov 10 19:56:26 2008
@@ -29,12 +29,12 @@
import org.apache.hama.DenseVector;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.Matrix;
-import org.apache.hama.Vector;
import org.apache.hama.io.VectorWritable;
import org.apache.hama.mapred.RowCyclicMap;
import org.apache.log4j.Logger;
-public class RowCyclicAdditionMap extends RowCyclicMap<IntWritable, VectorWritable> {
+public class RowCyclicAdditionMap extends
+ RowCyclicMap<IntWritable, VectorWritable> {
static final Logger LOG = Logger.getLogger(RowCyclicAdditionMap.class);
protected Matrix matrix_b;
public static final String MATRIX_B = "hama.addition.matrix.b";
@@ -64,9 +64,8 @@
OutputCollector<IntWritable, VectorWritable> output, Reporter reporter)
throws IOException {
- Vector v1 = matrix_b.getRow(key.get());
- output.collect(key, new VectorWritable(key.get(), (DenseVector) v1
- .add(value.getDenseVector())));
+ output.collect(key, new VectorWritable(key.get(),
+ (DenseVector) matrix_b.getRow(key.get()).add(value.getDenseVector())));
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java Mon Nov 10 19:56:26 2008
@@ -21,12 +21,10 @@
import java.io.IOException;
import java.util.Iterator;
-import java.util.Map;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.io.VectorEntry;
import org.apache.hama.io.VectorUpdate;
import org.apache.hama.io.VectorWritable;
import org.apache.hama.mapred.RowCyclicReduce;
@@ -39,11 +37,7 @@
throws IOException {
VectorUpdate update = new VectorUpdate(key.get());
- VectorWritable vector = values.next();
-
- for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
- update.put(f.getKey(), f.getValue().getValue());
- }
+ update.putAll(values.next().entrySet());
output.collect(key, update);
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Mon Nov 10 19:56:26 2008
@@ -29,7 +29,6 @@
import org.apache.hama.DenseVector;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.Matrix;
-import org.apache.hama.Vector;
import org.apache.hama.io.VectorWritable;
import org.apache.hama.mapred.RowCyclicMap;
import org.apache.log4j.Logger;
@@ -70,9 +69,7 @@
sum.clear();
for(int i = 0; i < value.size(); i++) {
- Vector v2 = matrix_b.getRow(i);
- DenseVector curr = (DenseVector) v2.scale(value.get(i).getValue());
- sum.add(curr);
+ sum.add(matrix_b.getRow(i).scale(value.get(i)));
}
output.collect(key, new VectorWritable(key.get(), sum));
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java Mon Nov 10 19:56:26 2008
@@ -21,12 +21,10 @@
import java.io.IOException;
import java.util.Iterator;
-import java.util.Map;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.io.VectorEntry;
import org.apache.hama.io.VectorUpdate;
import org.apache.hama.io.VectorWritable;
import org.apache.hama.mapred.RowCyclicReduce;
@@ -41,13 +39,8 @@
OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
throws IOException {
-
VectorUpdate update = new VectorUpdate(key.get());
- VectorWritable vector = values.next();
-
- for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
- update.put(f.getKey(), f.getValue().getValue());
- }
+ update.putAll(values.next().entrySet());
output.collect(key, update);
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java Mon Nov 10 19:56:26 2008
@@ -57,6 +57,14 @@
return new DenseVector(entries);
}
+ public Double get(Integer column) {
+ return this.entries.get(column).getValue();
+ }
+
+ public int size() {
+ return this.entries.size();
+ }
+
public VectorEntry put(Integer key, VectorEntry value) {
throw new UnsupportedOperationException("VectorWritable is read-only!");
}
@@ -124,17 +132,6 @@
}
/**
- * Get the VectorEntry that corresponds to column
- */
- public VectorEntry get(Integer column) {
- return this.entries.get(column);
- }
-
- public int size() {
- return this.entries.size();
- }
-
- /**
*
* The inner class for an entry of row.
*