You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/02/10 01:15:42 UTC
svn commit: r742798 - in /incubator/hama/trunk: CHANGES.txt
src/java/org/apache/hama/AbstractMatrix.java
src/java/org/apache/hama/DenseMatrix.java
src/test/org/apache/hama/TestDenseMatrix.java
Author: edwardyoon
Date: Tue Feb 10 00:15:41 2009
New Revision: 742798
URL: http://svn.apache.org/viewvc?rev=742798&view=rev
Log: (empty)
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Feb 10 00:15:41 2009
@@ -4,6 +4,7 @@
NEW FEATURES
+ HAMA-100: Implement of set(Matrix B) (edwardyoon)
HAMA-145: Add privacy policy page (edwardyoon)
HAMA-83: 2D sqaure blocking for dense matrix multiplication (edwardyoon)
HAMA-104: Add getNumMap/reduceTasks to HamaConfiguration (edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue Feb 10 00:15:41 2009
@@ -20,6 +20,7 @@
package org.apache.hama;
import java.io.IOException;
+import java.util.Map;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -31,9 +32,19 @@
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.mapred.IdentityTableReduce;
+import org.apache.hadoop.hbase.mapred.TableMap;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
import org.apache.hama.io.VectorUpdate;
import org.apache.hama.util.BytesUtil;
+import org.apache.hama.util.JobManager;
import org.apache.log4j.Logger;
/**
@@ -73,16 +84,16 @@
protected void create() throws IOException {
// It should run only when table doesn't exist.
if (!admin.tableExists(matrixPath)) {
- this.tableDesc.addFamily(new HColumnDescriptor(
- Bytes.toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
+ this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+ .toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
Integer.MAX_VALUE, HConstants.FOREVER, false));
this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
// It's a temporary data.
- this.tableDesc.addFamily(new HColumnDescriptor(
- Bytes.toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
- Integer.MAX_VALUE, HConstants.FOREVER, false));
-
+ this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+ .toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
+ Integer.MAX_VALUE, HConstants.FOREVER, false));
+
LOG.info("Initializing the matrix storage.");
this.admin.createTable(this.tableDesc);
LOG.info("Create Matrix " + matrixPath);
@@ -145,6 +156,45 @@
table.commit(update.getBatchUpdate());
}
+ /**
+ * Just full scan a table.
+ */
+ public static class TableReadMapper extends MapReduceBase implements
+ TableMap<ImmutableBytesWritable, BatchUpdate> {
+
+ @SuppressWarnings("unchecked")
+ public void map(ImmutableBytesWritable key, RowResult value,
+ OutputCollector<ImmutableBytesWritable, BatchUpdate> output,
+ @SuppressWarnings("unused")
+ Reporter reporter) throws IOException {
+
+ BatchUpdate update = new BatchUpdate(key.get());
+ for (Map.Entry<byte[], Cell> e : value.entrySet()) {
+ update.put(e.getKey(), e.getValue().getValue());
+ }
+ output.collect(key, update);
+ }
+ }
+
+ /** {@inheritDoc} */
+ public Matrix set(Matrix B) throws IOException {
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("set MR job : " + this.getPath());
+
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+ TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
+ + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
+ + Constants.BLOCK, TableReadMapper.class,
+ ImmutableBytesWritable.class, BatchUpdate.class, jobConf);
+ TableMapReduceUtil.initTableReduceJob(this.getPath(),
+ IdentityTableReduce.class, jobConf);
+
+ JobManager.execute(jobConf);
+ return this;
+ }
+
public String getRowLabel(int row) throws IOException {
Cell rows = null;
rows = table.get(BytesUtil.getRowIndex(row), Bytes
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Feb 10 00:15:41 2009
@@ -446,11 +446,6 @@
return null;
}
- public Matrix set(Matrix B) throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-
public void setRow(int row, Vector vector) throws IOException {
VectorUpdate update = new VectorUpdate(row);
update.putAll(((DenseVector) vector).getEntries().entrySet());
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Tue Feb 10 00:15:41 2009
@@ -29,6 +29,7 @@
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hama.io.DoubleEntry;
+import org.apache.hama.util.RandomVariable;
import org.apache.log4j.Logger;
/**
@@ -203,6 +204,17 @@
}
}
+ public void testSetMatrix() throws IOException {
+ Matrix a = new DenseMatrix(conf);
+ a.set(m1);
+
+ for (int i = 0; i < 5; i++) {
+ int x = RandomVariable.randInt(0, 10);
+ int y = RandomVariable.randInt(0, 10);
+ assertEquals(a.get(x, y), m1.get(x, y));
+ }
+ }
+
public void testLoadSave() throws IOException {
String path1 = m1.getPath();
// save m1 to aliase1