You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2008/09/16 07:08:12 UTC
svn commit: r695728 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/
src/java/org/apache/hama/algebra/ src/java/org/apache/hama/io/
src/java/org/apache/hama/mapred/
Author: edwardyoon
Date: Mon Sep 15 22:08:12 2008
New Revision: 695728
URL: http://svn.apache.org/viewvc?rev=695728&view=rev
Log: (empty)
Added:
incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Sep 15 22:08:12 2008
@@ -22,7 +22,8 @@
HAMA-2: The intial donation of Hama from the google project (edwardyoon)
IMPROVEMENTS
-
+
+ HAMA-58: Remove duplicated code (edwardyoon)
HAMA-56: Add setRow(int row, Vector vector) method to matrix inteface (edwardyoon)
HAMA-39: IO operations should throws an IOException (edwardyoon)
HAMA-52: Fixture setup (edwardyoon)
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Mon Sep 15 22:08:12 2008
@@ -26,12 +26,12 @@
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hama.io.VectorUpdate;
import org.apache.hama.util.Numeric;
import org.apache.log4j.Logger;
@@ -58,9 +58,9 @@
* @param conf configuration object
*/
public void setConfiguration(HamaConfiguration conf) {
- config = (HamaConfiguration) conf;
+ this.config = conf;
try {
- admin = new HBaseAdmin(config);
+ this.admin = new HBaseAdmin(config);
} catch (MasterNotRunningException e) {
LOG.error(e, e);
}
@@ -71,10 +71,10 @@
*/
protected void create() {
try {
- tableDesc.addFamily(new HColumnDescriptor(Constants.METADATA));
- tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.METADATA));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
LOG.info("Initializing the matrix storage.");
- admin.createTable(tableDesc);
+ this.admin.createTable(this.tableDesc);
} catch (IOException e) {
LOG.error(e, e);
}
@@ -116,9 +116,9 @@
/** {@inheritDoc} */
public void set(int i, int j, double value) throws IOException {
- BatchUpdate b = new BatchUpdate(Numeric.intToBytes(i));
- b.put(Numeric.getColumnIndex(j), Numeric.doubleToBytes(value));
- table.commit(b);
+ VectorUpdate update = new VectorUpdate(i);
+ update.put(j, value);
+ table.commit(update.getBatchUpdate());
}
/** {@inheritDoc} */
@@ -128,11 +128,11 @@
/** {@inheritDoc} */
public void setDimension(int rows, int columns) throws IOException {
- BatchUpdate b = new BatchUpdate(Constants.METADATA);
- b.put(Constants.METADATA_ROWS, Numeric.intToBytes(rows));
- b.put(Constants.METADATA_COLUMNS, Numeric.intToBytes(columns));
+ VectorUpdate update = new VectorUpdate(Constants.METADATA);
+ update.put(Constants.METADATA_ROWS, rows);
+ update.put(Constants.METADATA_COLUMNS, columns);
- table.commit(b);
+ table.commit(update.getBatchUpdate());
}
public String getRowAttribute(int row) throws IOException {
@@ -144,10 +144,9 @@
}
public void setRowAttribute(int row, String name) throws IOException {
- BatchUpdate b = new BatchUpdate(Numeric.intToBytes(row));
- b.put(Constants.ATTRIBUTE + "string", Bytes.toBytes(name));
-
- table.commit(b);
+ VectorUpdate update = new VectorUpdate(row);
+ update.put(Constants.ATTRIBUTE + "string", name);
+ table.commit(update.getBatchUpdate());
}
/** {@inheritDoc} */
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Mon Sep 15 22:08:12 2008
@@ -20,14 +20,12 @@
package org.apache.hama;
import java.io.IOException;
-import java.util.Map;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scanner;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.JobConf;
@@ -37,6 +35,7 @@
import org.apache.hama.algebra.MultiplicationReduce;
import org.apache.hama.io.VectorEntry;
import org.apache.hama.io.VectorMapWritable;
+import org.apache.hama.io.VectorUpdate;
import org.apache.hama.mapred.MatrixReduce;
import org.apache.hama.util.Numeric;
import org.apache.hama.util.RandomVariable;
@@ -206,14 +205,10 @@
}
public void setRow(int row, Vector vector) throws IOException {
- BatchUpdate b = new BatchUpdate(Numeric.intToBytes(row));
- for (Map.Entry<Integer, VectorEntry> e : ((DenseVector) vector)
- .getEntries().entrySet()) {
- b.put(Numeric.getColumnIndex(e.getKey()), Numeric.doubleToBytes(e
- .getValue().getValue()));
- }
+ VectorUpdate update = new VectorUpdate(row);
+ update.putAll(((DenseVector) vector).getEntries().entrySet());
- table.commit(b);
+ table.commit(update.getBatchUpdate());
}
public void setColumn(int column, Vector vector) throws IOException {
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java Mon Sep 15 22:08:12 2008
@@ -23,29 +23,29 @@
import java.util.Iterator;
import java.util.Map;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hama.DenseVector;
import org.apache.hama.io.VectorEntry;
+import org.apache.hama.io.VectorUpdate;
import org.apache.hama.mapred.MatrixReduce;
-import org.apache.hama.util.Numeric;
public class AdditionReduce extends MatrixReduce<IntWritable, DenseVector> {
@Override
public void reduce(IntWritable key, Iterator<DenseVector> values,
- OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+ OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
throws IOException {
- BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+ VectorUpdate update = new VectorUpdate(key.get());
DenseVector vector = values.next();
+
for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
- b.put(Numeric.getColumnIndex(f.getKey()), Numeric.doubleToBytes(f.getValue().getValue()));
+ update.put(f.getKey(), f.getValue().getValue());
}
- output.collect(key, b);
+ output.collect(key, update);
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java Mon Sep 15 22:08:12 2008
@@ -24,13 +24,12 @@
import java.util.Iterator;
import java.util.Map;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hama.DenseVector;
+import org.apache.hama.io.VectorUpdate;
import org.apache.hama.mapred.MatrixReduce;
-import org.apache.hama.util.Numeric;
import org.apache.log4j.Logger;
public class MultiplicationReduce extends
@@ -39,10 +38,11 @@
@Override
public void reduce(IntWritable key, Iterator<DenseVector> values,
- OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+ OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
throws IOException {
- BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+ VectorUpdate update = new VectorUpdate(key.get());
+
DenseVector sum;
Map<Integer, Double> buffer = new HashMap<Integer, Double>();
@@ -57,13 +57,8 @@
}
}
}
-
- for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
- byte[] value = Numeric.doubleToBytes(f.getValue());
- b.put(Numeric.getColumnIndex(f.getKey()), value);
- }
-
- output.collect(key, b);
+ update.putAll(buffer);
+ output.collect(key, update);
}
}
Added: incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java?rev=695728&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java Mon Sep 15 22:08:12 2008
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.io;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hama.util.Numeric;
+
+public class VectorUpdate {
+ private BatchUpdate batchUpdate;
+
+ public VectorUpdate(int i) {
+ this.batchUpdate = new BatchUpdate(Numeric.intToBytes(i));
+ }
+
+ public VectorUpdate(String metadata) {
+ this.batchUpdate = new BatchUpdate(metadata);
+ }
+
+ public void put(int j, double value) {
+ this.batchUpdate.put(Numeric.getColumnIndex(j), Numeric
+ .doubleToBytes(value));
+ }
+
+ public void put(String column, String val) {
+ this.batchUpdate.put(column, Bytes.toBytes(val));
+ }
+
+ public void put(String metadataRows, int rows) {
+ this.batchUpdate.put(metadataRows, Numeric.intToBytes(rows));
+ }
+
+ public BatchUpdate getBatchUpdate() {
+ return this.batchUpdate;
+ }
+
+ public void putAll(Map<Integer, Double> buffer) {
+ for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
+ put(f.getKey(), f.getValue());
+ }
+ }
+
+ public void putAll(Set<Entry<Integer, VectorEntry>> entrySet) {
+ for (Map.Entry<Integer, VectorEntry> e : entrySet) {
+ put(e.getKey(), e.getValue().getValue());
+ }
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java Mon Sep 15 22:08:12 2008
@@ -26,7 +26,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -35,9 +34,10 @@
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
+import org.apache.hama.io.VectorUpdate;
public class MatrixOutputFormat extends
- FileOutputFormat<IntWritable, BatchUpdate> {
+ FileOutputFormat<IntWritable, VectorUpdate> {
/** JobConf parameter that specifies the output table */
public static final String OUTPUT_TABLE = "hama.mapred.output";
@@ -48,7 +48,7 @@
* and write to an HBase table
*/
protected static class TableRecordWriter implements
- RecordWriter<IntWritable, BatchUpdate> {
+ RecordWriter<IntWritable, VectorUpdate> {
private HTable m_table;
/**
@@ -65,8 +65,8 @@
}
/** {@inheritDoc} */
- public void write(IntWritable key, BatchUpdate value) throws IOException {
- m_table.commit(value);
+ public void write(IntWritable key, VectorUpdate value) throws IOException {
+ m_table.commit(value.getBatchUpdate());
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java Mon Sep 15 22:08:12 2008
@@ -31,10 +31,11 @@
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorUpdate;
@SuppressWarnings("unchecked")
public abstract class MatrixReduce<K extends WritableComparable, V extends Writable>
- extends MapReduceBase implements Reducer<K, V, IntWritable, BatchUpdate> {
+ extends MapReduceBase implements Reducer<K, V, IntWritable, VectorUpdate> {
/**
* Use this before submitting a TableReduce job. It will
* appropriately set up the JobConf.
@@ -61,6 +62,6 @@
* @throws IOException
*/
public abstract void reduce(K key, Iterator<V> values,
- OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+ OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
throws IOException;
}
\ No newline at end of file