You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/10/26 09:04:11 UTC
svn commit: r829720 - in /incubator/hama/trunk: ./
src/examples/org/apache/hama/examples/ src/java/org/apache/hama/
src/java/org/apache/hama/io/ src/java/org/apache/hama/mapred/
src/java/org/apache/hama/matrix/ src/java/org/apache/hama/matrix/algebra/ ...
Author: edwardyoon
Date: Mon Oct 26 08:04:09 2009
New Revision: 829720
URL: http://svn.apache.org/viewvc?rev=829720&view=rev
Log:
Removing hama shell from version 0.2
Removed:
incubator/hama/trunk/src/java/org/apache/hama/shell/
incubator/hama/trunk/src/test/org/apache/hama/shell/
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/build.xml
incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/Constants.java
incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java
incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractVector.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseVector.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/SparseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/JacobiEigenValue.java
incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Oct 26 08:04:09 2009
@@ -42,6 +42,7 @@
IMPROVEMENTS
+ HAMA-215: Removing hama shell from version 0.2 (edwardyoon)
HAMA-204: Replacement of TransposeMap/Reduce (edwardyoon)
HAMA-213: Replacement of TableReadMapper (edwardyoon)
HAMA-203: Replacement of RandomMatrixMap/Reduce (edwardyoon)
Modified: incubator/hama/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/build.xml?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/build.xml (original)
+++ incubator/hama/trunk/build.xml Mon Oct 26 08:04:09 2009
@@ -29,7 +29,6 @@
<property file="${basedir}/build.properties" />
<property name="src.dir" location="${basedir}/src/java" />
- <property name="src.gen.dir" location="${basedir}/src-gen" />
<property name="src.test" location="${basedir}/src/test" />
<property name="src.examples" location="${basedir}/src/examples" />
@@ -41,9 +40,6 @@
<!-- javacc properties -->
<property name="javacc.home" value="${basedir}/lib" />
- <property name="src.gen.parser.expression.dir" value="${src.gen.dir}/org/apache/hama/shell/parser/expression" />
- <property name="src.gen.parser.script.dir" value="${src.gen.dir}/org/apache/hama/shell/parser/script" />
-
<!-- build properties -->
<property name="test.output" value="no" />
@@ -89,9 +85,6 @@
classname="edu.umd.cs.findbugs.anttask.FindBugsTask" />
<target name="init">
- <mkdir dir="${src.gen.dir}" />
- <mkdir dir="${src.gen.parser.script.dir}" />
- <mkdir dir="${src.gen.parser.expression.dir}" />
<mkdir dir="${build.dir}" />
<mkdir dir="${build.classes}" />
<mkdir dir="${build.test}" />
@@ -108,17 +101,12 @@
<!-- ================================================================== -->
<!-- Java Compiler Compiler, generate Parsers -->
<!-- ================================================================== -->
- <target name="cc-compile" depends="init" description="Create and Compile Parser">
- <jjtree target="${src.dir}/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt" outputdirectory="${src.gen.parser.expression.dir}" javacchome="${javacc.home}" />
- <javacc target="${src.gen.parser.expression.dir}/HamaExpressionParser.jj" outputdirectory="${src.gen.parser.expression.dir}" javacchome="${javacc.home}" />
- <javacc target="${src.dir}/org/apache/hama/shell/parser/script/HamaScriptParser.jj" outputdirectory="${src.gen.parser.script.dir}" javacchome="${javacc.home}" />
- </target>
- <target name="compile" depends="init, cc-compile">
+ <target name="compile" depends="init">
<!--Compile whats under src and generated java classes made from jsp-->
<mkdir dir="${build.src}" />
<javac encoding="${build.encoding}"
- srcdir="${src.dir};${src.gen.dir};${build.src}" includes="**/*.java"
+ srcdir="${src.dir};${build.src}" includes="**/*.java"
destdir="${build.classes}" debug="${javac.debug}"
deprecation="${javac.deprecation}">
<classpath refid="classpath" />
@@ -138,10 +126,6 @@
<target name="jar" depends="compile">
<jar jarfile="${build.dir}/${final.name}.jar"
basedir="${build.classes}">
- <manifest>
- <attribute name="Main-Class"
- value="org/apache/hama/shell/HamaShell" />
- </manifest>
</jar>
</target>
@@ -242,7 +226,7 @@
</target>
<!-- Javadoc -->
- <target name="javadoc" depends="cc-compile" description="Generate javadoc">
+ <target name="javadoc" description="Generate javadoc">
<mkdir dir="${build.javadoc}" />
<javadoc overview="${src.dir}/overview.html" packagenames="org.apache.hama.*"
destdir="${build.javadoc}" author="true" version="true" use="true"
@@ -251,9 +235,6 @@
<packageset dir="${src.dir}">
<include name="org/apache/**" />
</packageset>
- <packageset dir="${src.gen.dir}">
- <include name="org/apache/**" />
- </packageset>
<link href="${javadoc.link.java}" />
<classpath>
<path refid="classpath" />
@@ -318,7 +299,6 @@
<!-- Clean. Delete the build files, and their directories -->
<!-- ================================================================== -->
<target name="clean">
- <delete dir="${src.gen.dir}" />
<delete dir="${build.dir}" />
</target>
</project>
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java Mon Oct 26 08:04:09 2009
@@ -49,7 +49,7 @@
if (BytesUtil.getRowIndex(key.get()) == i) {
dotProduct = 0;
}
- put.add(Bytes.toBytes(Constants.COLUMN_FAMILY), Bytes.toBytes(String
+ put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String
.valueOf(i)), BytesUtil.doubleToBytes(dotProduct));
}
@@ -82,7 +82,7 @@
job.getConfiguration().set("input.matrix", admin.getPath(args[0]));
Scan scan = new Scan();
- scan.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ scan.addFamily(Constants.COLUMNFAMILY);
TableMapReduceUtil.initTableMapperJob(admin.getPath(args[0]), scan,
ComputeSimilarityMapper.class, ImmutableBytesWritable.class, Put.class, job);
Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Mon Oct 26 08:04:09 2009
@@ -19,6 +19,7 @@
*/
package org.apache.hama;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
/**
@@ -33,7 +34,7 @@
public final static String CINDEX = "cIndex";
/** The attribute column family */
- public final static String ATTRIBUTE = "attribute";
+ public static byte[] ATTRIBUTE = Bytes.toBytes("attribute");
/** The type of the matrix */
public final static String METADATA_TYPE = "type";
@@ -60,7 +61,7 @@
@Deprecated
public final static String COLUMN = "column:";
- public final static String COLUMN_FAMILY = "column";
+ public static byte[] COLUMNFAMILY = Bytes.toBytes("column");
/** Temporary random matrices name prefix */
public final static String RANDOM = "rand";
Modified: incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java Mon Oct 26 08:04:09 2009
@@ -151,9 +151,9 @@
HTable matrix = new HTable(conf, tableName);
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
+ get.addFamily(Constants.ATTRIBUTE);
byte[] result = matrix.get(get).getValue(
- Bytes.toBytes(Constants.ATTRIBUTE),
+ Constants.ATTRIBUTE,
Bytes.toBytes(Constants.METADATA_REFERENCE));
return (result == null) ? 0 : Bytes.toInt(result);
@@ -224,9 +224,9 @@
HTable table = new HTable(conf, path);
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
+ get.addFamily(Constants.ATTRIBUTE);
byte[] result = table.get(get).getValue(
- Bytes.toBytes(Constants.ATTRIBUTE), Bytes.toBytes("type"));
+ Constants.ATTRIBUTE, Bytes.toBytes("type"));
return Bytes.toString(result);
} catch (IOException e) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java Mon Oct 26 08:04:09 2009
@@ -52,8 +52,8 @@
public void put(int j, double value) {
this.batchUpdate.put(BytesUtil.getColumnIndex(j), BytesUtil
.doubleToBytes(value));
- this.put.add(Bytes.toBytes(Constants.COLUMN_FAMILY), Bytes.toBytes(String
- .valueOf(j)), BytesUtil.doubleToBytes(value));
+ this.put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j)),
+ BytesUtil.doubleToBytes(value));
}
/**
@@ -73,16 +73,18 @@
@Deprecated
public void put(int j, String name) {
- this.batchUpdate.put(Bytes.toBytes((Constants.ATTRIBUTE + j)), Bytes
+ this.batchUpdate.put(Bytes
+ .toBytes((Bytes.toString(Constants.ATTRIBUTE) + j)), Bytes
.toBytes(name));
}
public void put(String j, String val) {
this.batchUpdate.put(j, Bytes.toBytes(val));
}
-
+
public void put(String column, String qualifier, String val) {
- this.put.add(Bytes.toBytes(column), Bytes.toBytes(qualifier), Bytes.toBytes(val));
+ this.put.add(Bytes.toBytes(column), Bytes.toBytes(qualifier), Bytes
+ .toBytes(val));
}
public void put(String row, int val) {
Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/VectorInputFormat.java Mon Oct 26 08:04:09 2009
@@ -20,10 +20,12 @@
package org.apache.hama.mapred;
import java.io.IOException;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.mapred.TableSplit;
import org.apache.hadoop.hbase.util.Bytes;
@@ -37,7 +39,7 @@
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hama.matrix.DenseVector;
+import org.apache.hama.io.DoubleEntry;
import org.apache.hama.util.BytesUtil;
public class VectorInputFormat extends HTableInputFormatBase implements
@@ -116,7 +118,14 @@
byte[] row = result.getRow();
key.set(BytesUtil.bytesToInt(row));
lastRow = row;
- Writables.copyWritable(new DenseVector(result).getEntries(), value);
+
+ MapWritable mw = new MapWritable();
+ for (Map.Entry<byte[], Cell> f : result.entrySet()) {
+ mw.put(new IntWritable(BytesUtil.getColumnIndex(f.getKey())),
+ new DoubleEntry(f.getValue()));
+ }
+
+ Writables.copyWritable(mw, value);
processedRows++;
}
return hasMore;
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractMatrix.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractMatrix.java Mon Oct 26 08:04:09 2009
@@ -141,10 +141,8 @@
protected void create() throws IOException {
// It should run only when table doesn't exist.
if (!admin.tableExists(matrixPath)) {
- this.tableDesc.addFamily(new HColumnDescriptor(Bytes
- .toBytes(Constants.COLUMN_FAMILY)));
- this.tableDesc.addFamily(new HColumnDescriptor(Bytes
- .toBytes(Constants.ATTRIBUTE)));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMNFAMILY));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
this.tableDesc.addFamily(new HColumnDescriptor(Bytes
.toBytes(Constants.ALIASEFAMILY)));
@@ -169,7 +167,7 @@
// Record the matrix type in METADATA_TYPE
Put put = new Put(Bytes.toBytes(Constants.METADATA));
- put.add(Bytes.toBytes(Constants.ATTRIBUTE), Bytes
+ put.add(Constants.ATTRIBUTE, Bytes
.toBytes(Constants.METADATA_TYPE), Bytes.toBytes(this.getClass()
.getSimpleName()));
table.put(put);
@@ -332,8 +330,8 @@
/** {@inheritDoc} */
public int getRows() throws IOException {
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes("rows"));
return (result != null) ? BytesUtil.bytesToInt(result) : 0;
@@ -342,8 +340,8 @@
/** {@inheritDoc} */
public int getColumns() throws IOException {
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes("columns"));
return BytesUtil.bytesToInt(result);
@@ -352,8 +350,8 @@
/** {@inheritDoc} */
public String getRowLabel(int row) throws IOException {
Get get = new Get(BytesUtil.getRowIndex(row));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes("string"));
return (result != null) ? Bytes.toString(result) : null;
@@ -361,13 +359,8 @@
/** {@inheritDoc} */
public void setColumnLabel(int column, String name) throws IOException {
- /*
- * VectorUpdate update = new VectorUpdate(Constants.CINDEX);
- * update.put(column, name); table.commit(update.getBatchUpdate());
- */
-
Put put = new Put(Bytes.toBytes(Constants.CINDEX));
- put.add(Bytes.toBytes(Constants.ATTRIBUTE), Bytes.toBytes(String
+ put.add(Constants.ATTRIBUTE, Bytes.toBytes(String
.valueOf(column)), Bytes.toBytes(name));
table.put(put);
}
@@ -375,8 +368,8 @@
/** {@inheritDoc} */
public String getColumnLabel(int column) throws IOException {
Get get = new Get(Bytes.toBytes(Constants.CINDEX));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes(String.valueOf(column)));
return (result != null) ? Bytes.toString(result) : null;
@@ -385,16 +378,15 @@
/** {@inheritDoc} */
public void setRowLabel(int row, String name) throws IOException {
VectorUpdate update = new VectorUpdate(row);
- update.put(Constants.ATTRIBUTE, "string", name);
+ update.put(Bytes.toString(Constants.ATTRIBUTE), "string", name);
table.put(update.getPut());
}
/** {@inheritDoc} */
public void setDimension(int rows, int columns) throws IOException {
Put put = new Put(Bytes.toBytes(Constants.METADATA));
- byte[] family = Bytes.toBytes(Constants.ATTRIBUTE);
- put.add(family, Bytes.toBytes("rows"), BytesUtil.intToBytes(rows));
- put.add(family, Bytes.toBytes("columns"), BytesUtil.intToBytes(columns));
+ put.add(Constants.ATTRIBUTE, Bytes.toBytes("rows"), BytesUtil.intToBytes(rows));
+ put.add(Constants.ATTRIBUTE, Bytes.toBytes("columns"), BytesUtil.intToBytes(columns));
table.put(put);
}
@@ -425,7 +417,7 @@
if (alpha.equals(new Double(1))) {
put.add(family, qualifier, val);
} else {
- if (Bytes.toString(family).equals(Constants.COLUMN_FAMILY)) {
+ if (Bytes.toString(family).equals(Bytes.toString(Constants.COLUMNFAMILY))) {
double currVal = BytesUtil.bytesToDouble(val);
put.add(family, qualifier, BytesUtil.doubleToBytes(currVal * alpha));
} else {
@@ -456,8 +448,8 @@
Job job = new Job(config, "set MR job : " + this.getPath());
Scan scan = new Scan();
- scan.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
- scan.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
+ scan.addFamily(Constants.COLUMNFAMILY);
+ scan.addFamily(Constants.ATTRIBUTE);
scan.addFamily(Bytes.toBytes(Constants.ALIASEFAMILY));
scan.addFamily(Bytes.toBytes(Constants.BLOCK_FAMILY));
scan.addFamily(Bytes.toBytes(JacobiEigenValue.EI_COLUMNFAMILY));
@@ -485,8 +477,8 @@
Job job = new Job(config, "set MR job : " + this.getPath());
Scan scan = new Scan();
- scan.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
- scan.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
+ scan.addFamily(Constants.COLUMNFAMILY);
+ scan.addFamily(Constants.ATTRIBUTE);
scan.addFamily(Bytes.toBytes(Constants.ALIASEFAMILY));
scan.addFamily(Bytes.toBytes(Constants.BLOCK_FAMILY));
scan.addFamily(Bytes.toBytes(JacobiEigenValue.EI_COLUMNFAMILY));
@@ -518,7 +510,7 @@
protected void setReference(int reference) throws IOException {
Put put = new Put(Bytes.toBytes(Constants.METADATA));
- put.add(Bytes.toBytes(Constants.ATTRIBUTE), Bytes
+ put.add(Constants.ATTRIBUTE, Bytes
.toBytes(Constants.METADATA_REFERENCE), Bytes.toBytes(reference));
table.put(put);
}
@@ -527,8 +519,8 @@
int reference = 1;
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes(Constants.METADATA_REFERENCE));
if (result != null) {
@@ -543,8 +535,8 @@
int reference = 0;
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = table.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes(Constants.METADATA_REFERENCE));
if (result != null) {
@@ -599,7 +591,7 @@
Job job = new Job(config, "set MR job : " + this.getPath());
Scan scan = new Scan();
- scan.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ scan.addFamily(Constants.COLUMNFAMILY);
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(
this.getPath(), scan, TransposeMap.class, IntWritable.class,
MapWritable.class, job);
@@ -622,7 +614,7 @@
Put put = new Put(Bytes.toBytes(Constants.METADATA));
put.add(Bytes.toBytes(Constants.ALIASEFAMILY), Bytes.toBytes("name"), Bytes
.toBytes(aliasename));
- put.add(Bytes.toBytes(Constants.ATTRIBUTE), Bytes.toBytes("type"), Bytes
+ put.add(Constants.ATTRIBUTE, Bytes.toBytes("type"), Bytes
.toBytes(this.getType()));
table.put(put);
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractVector.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/AbstractVector.java Mon Oct 26 08:04:09 2009
@@ -43,18 +43,9 @@
static final Logger LOG = Logger.getLogger(AbstractVector.class);
protected MapWritable entries;
- @Deprecated
- public void initMap(RowResult row) {
- this.entries = new MapWritable();
- for (Map.Entry<byte[], Cell> f : row.entrySet()) {
- this.entries.put(new IntWritable(BytesUtil.getColumnIndex(f.getKey())),
- new DoubleEntry(f.getValue()));
- }
- }
-
public void initMap(Result rs) {
this.entries = new MapWritable();
- NavigableMap<byte[], byte[]> map = rs.getFamilyMap(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ NavigableMap<byte[], byte[]> map = rs.getFamilyMap(Constants.COLUMNFAMILY);
for (Map.Entry<byte[], byte[]> e : map.entrySet()) {
if(e != null) {
this.entries.put(new IntWritable(Integer.valueOf(Bytes.toString(e.getKey()))),
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseMatrix.java Mon Oct 26 08:04:09 2009
@@ -22,22 +22,17 @@
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
-import java.util.NavigableMap;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Scanner;
-import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.DoubleWritable;
@@ -47,14 +42,13 @@
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hama.Constants;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.io.BlockID;
@@ -250,19 +244,18 @@
final Path inDir = new Path(TMP_DIR, "in");
FileInputFormat.setInputPaths(job, inDir);
job.setMapperClass(RandomMatrixMapper.class);
-
+
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(MapWritable.class);
-
+
job.getConfiguration().setInt("matrix.column", n);
job.getConfiguration().set("matrix.type", TABLE_PREFIX);
job.getConfiguration().set("matrix.density", "100");
-
job.setInputFormatClass(SequenceFileInputFormat.class);
final FileSystem fs = FileSystem.get(job.getConfiguration());
int interval = m / conf.getNumMapTasks();
-
+
// generate an input file for each map task
for (int i = 0; i < conf.getNumMapTasks(); ++i) {
final Path file = new Path(inDir, "part" + i);
@@ -273,8 +266,9 @@
} else {
end = new IntWritable(m - 1);
}
- final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job.getConfiguration(),
- file, IntWritable.class, IntWritable.class, CompressionType.NONE);
+ final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job
+ .getConfiguration(), file, IntWritable.class, IntWritable.class,
+ CompressionType.NONE);
try {
writer.append(start, end);
} finally {
@@ -288,7 +282,7 @@
job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, rand.getPath());
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Writable.class);
-
+
try {
job.waitForCompletion(true);
} catch (InterruptedException e) {
@@ -341,9 +335,8 @@
throw new ArrayIndexOutOfBoundsException(i + ", " + j);
Get get = new Get(BytesUtil.getRowIndex(i));
- get.addColumn(Bytes.toBytes(Constants.COLUMN_FAMILY));
- byte[] result = table.get(get).getValue(
- Bytes.toBytes(Constants.COLUMN_FAMILY),
+ get.addColumn(Constants.COLUMNFAMILY);
+ byte[] result = table.get(get).getValue(Constants.COLUMNFAMILY,
Bytes.toBytes(String.valueOf(j)));
if (result == null)
@@ -361,7 +354,7 @@
*/
public DenseVector getRow(int i) throws IOException {
Get get = new Get(BytesUtil.getRowIndex(i));
- get.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ get.addFamily(Constants.COLUMNFAMILY);
Result r = table.get(get);
return new DenseVector(r);
}
@@ -375,17 +368,16 @@
*/
public DenseVector getColumn(int j) throws IOException {
Scan scan = new Scan();
- scan.addColumn(Bytes.toBytes(Constants.COLUMN_FAMILY)
- , Bytes.toBytes(String.valueOf(j)));
+ scan.addColumn(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j)));
ResultScanner s = table.getScanner(scan);
Result r = null;
-
+
MapWritable trunk = new MapWritable();
while ((r = s.next()) != null) {
- byte[] value = r.getValue(Bytes.toBytes(Constants.COLUMN_FAMILY)
- , Bytes.toBytes(String.valueOf(j)));
+ byte[] value = r.getValue(Constants.COLUMNFAMILY, Bytes.toBytes(String
+ .valueOf(j)));
LOG.info(Bytes.toString(r.getRow()));
- trunk.put(new IntWritable(BytesUtil.getRowIndex(r.getRow())),
+ trunk.put(new IntWritable(BytesUtil.getRowIndex(r.getRow())),
new DoubleEntry(value));
}
@@ -655,25 +647,29 @@
int columnSize = (j1 - j0) + 1;
SubMatrix result = new SubMatrix((i1 - i0) + 1, columnSize);
- byte[][] cols = new byte[columnSize][];
+ Scan scan = new Scan();
for (int j = j0, jj = 0; j <= j1; j++, jj++) {
- cols[jj] = BytesUtil.getColumnIndex(j);
+ scan.addColumn(Constants.COLUMNFAMILY, Bytes
+ .toBytes(String.valueOf(j)));
}
+ scan.setStartRow(BytesUtil.getRowIndex(i0));
+ scan.setStopRow(BytesUtil.getRowIndex(i1 + 1));
+ ResultScanner s = table.getScanner(scan);
+ Iterator<Result> it = s.iterator();
- Scanner scan = table.getScanner(cols, BytesUtil.getRowIndex(i0), BytesUtil
- .getRowIndex(i1 + 1));
- Iterator<RowResult> it = scan.iterator();
int i = 0;
- RowResult rs = null;
+ Result rs = null;
while (it.hasNext()) {
rs = it.next();
for (int j = j0, jj = 0; j <= j1; j++, jj++) {
- result.set(i, jj, rs.get(BytesUtil.getColumnIndex(j)).getValue());
+ byte[] vv = rs.getValue(Constants.COLUMNFAMILY, Bytes
+ .toBytes(String.valueOf(j)));
+ System.out.println(BytesUtil.bytesToDouble(vv));
+ result.set(i, jj, vv);
}
i++;
}
- scan.close();
return result;
}
@@ -779,7 +775,8 @@
jobConf.setInputFormat(JacobiEigenValue.PivotInputFormat.class);
jobConf.set(JacobiEigenValue.PivotInputFormat.COLUMN_LIST,
JacobiEigenValue.EIIND);
- org.apache.hadoop.mapred.FileInputFormat.addInputPaths(jobConf, getPath());
+ org.apache.hadoop.mapred.FileInputFormat
+ .addInputPaths(jobConf, getPath());
jobConf.setMapOutputKeyClass(Pair.class);
jobConf.setMapOutputValueClass(DoubleWritable.class);
@@ -862,10 +859,12 @@
jobConf.setInputFormat(JacobiEigenValue.RotationInputFormat.class);
jobConf.set(JacobiEigenValue.RotationInputFormat.COLUMN_LIST,
JacobiEigenValue.EIIND);
- org.apache.hadoop.mapred.FileInputFormat.addInputPaths(jobConf, getPath());
+ org.apache.hadoop.mapred.FileInputFormat
+ .addInputPaths(jobConf, getPath());
jobConf.setMapOutputKeyClass(NullWritable.class);
jobConf.setMapOutputValueClass(NullWritable.class);
- org.apache.hadoop.mapred.FileInputFormat.addInputPaths(jobConf, getPath());
+ org.apache.hadoop.mapred.FileInputFormat
+ .addInputPaths(jobConf, getPath());
jobConf.setOutputFormat(NullOutputFormat.class);
JobManager.execute(jobConf);
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseVector.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/DenseVector.java Mon Oct 26 08:04:09 2009
@@ -25,7 +25,6 @@
import java.util.Set;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Writable;
@@ -42,10 +41,6 @@
public DenseVector() {
this(new MapWritable());
}
-
- public DenseVector(RowResult row) {
- this.initMap(row);
- }
public DenseVector(MapWritable m) {
this.entries = m;
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/SparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/SparseMatrix.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/SparseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/SparseMatrix.java Mon Oct 26 08:04:09 2009
@@ -125,19 +125,18 @@
final Path inDir = new Path(TMP_DIR, "in");
FileInputFormat.setInputPaths(job, inDir);
job.setMapperClass(RandomMatrixMapper.class);
-
+
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(MapWritable.class);
-
+
job.getConfiguration().setInt("matrix.column", n);
job.getConfiguration().set("matrix.type", TABLE_PREFIX);
job.getConfiguration().set("matrix.density", String.valueOf(percent));
-
job.setInputFormatClass(SequenceFileInputFormat.class);
final FileSystem fs = FileSystem.get(job.getConfiguration());
int interval = m / conf.getNumMapTasks();
-
+
// generate an input file for each map task
for (int i = 0; i < conf.getNumMapTasks(); ++i) {
final Path file = new Path(inDir, "part" + i);
@@ -148,8 +147,9 @@
} else {
end = new IntWritable(m - 1);
}
- final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job.getConfiguration(),
- file, IntWritable.class, IntWritable.class, CompressionType.NONE);
+ final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job
+ .getConfiguration(), file, IntWritable.class, IntWritable.class,
+ CompressionType.NONE);
try {
writer.append(start, end);
} finally {
@@ -163,7 +163,7 @@
job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, rand.getPath());
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Writable.class);
-
+
try {
job.waitForCompletion(true);
} catch (InterruptedException e) {
@@ -195,8 +195,8 @@
throw new ArrayIndexOutOfBoundsException(i + ", " + j);
Get get = new Get(BytesUtil.getRowIndex(i));
- get.addColumn(Bytes.toBytes(Constants.COLUMN_FAMILY));
- byte[] result = table.get(get).getValue(Bytes.toBytes(Constants.COLUMN_FAMILY),
+ get.addColumn(Constants.COLUMNFAMILY);
+ byte[] result = table.get(get).getValue(Constants.COLUMNFAMILY,
Bytes.toBytes(String.valueOf(j)));
return (result != null) ? BytesUtil.bytesToDouble(result) : 0.0;
}
@@ -216,7 +216,7 @@
*/
public SparseVector getRow(int i) throws IOException {
Get get = new Get(BytesUtil.getRowIndex(i));
- get.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ get.addFamily(Constants.COLUMNFAMILY);
Result r = table.get(get);
// return new SparseVector(r.getRowResult());
return new SparseVector(r);
Modified: incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/JacobiEigenValue.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/JacobiEigenValue.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/JacobiEigenValue.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/matrix/algebra/JacobiEigenValue.java Mon Oct 26 08:04:09 2009
@@ -174,8 +174,8 @@
super.init();
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = htable.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = htable.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes("rows"));
size = (result != null) ? BytesUtil.bytesToInt(result) : 0;
@@ -409,8 +409,8 @@
super.init();
Get get = new Get(Bytes.toBytes(Constants.METADATA));
- get.addFamily(Bytes.toBytes(Constants.ATTRIBUTE));
- byte[] result = htable.get(get).getValue(Bytes.toBytes(Constants.ATTRIBUTE),
+ get.addFamily(Constants.ATTRIBUTE);
+ byte[] result = htable.get(get).getValue(Constants.ATTRIBUTE,
Bytes.toBytes("rows"));
size = (result != null) ? BytesUtil.bytesToInt(result) : 0;
Modified: incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java?rev=829720&r1=829719&r2=829720&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java Mon Oct 26 08:04:09 2009
@@ -52,7 +52,7 @@
job.getConfiguration().set("input.matrix", m1.getPath());
Scan scan = new Scan();
- scan.addFamily(Bytes.toBytes(Constants.COLUMN_FAMILY));
+ scan.addFamily(Constants.COLUMNFAMILY);
TableMapReduceUtil.initTableMapperJob(m1.getPath(), scan,
ComputeSimilarity.class, ImmutableBytesWritable.class, Put.class, job);
@@ -87,7 +87,7 @@
if (BytesUtil.getRowIndex(key.get()) == i) {
dotProduct = 0;
}
- put.add(Bytes.toBytes(Constants.COLUMN_FAMILY), Bytes.toBytes(String
+ put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String
.valueOf(i)), BytesUtil.doubleToBytes(dotProduct));
}