You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/02/24 03:52:49 UTC
svn commit: r747265 - in /incubator/hama/trunk/src: java/org/apache/hama/
java/org/apache/hama/algebra/ test/org/apache/hama/
Author: edwardyoon
Date: Tue Feb 24 02:52:48 2009
New Revision: 747265
URL: http://svn.apache.org/viewvc?rev=747265&view=rev
Log:
Add SparseMatrix/Vector classes
Added:
incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/SparseVector.java
incubator/hama/trunk/src/test/org/apache/hama/TestSparseMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java
Modified:
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue Feb 24 02:52:48 2009
@@ -1,360 +1,384 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.RegionException;
-import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.io.Cell;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.RowResult;
-import org.apache.hadoop.hbase.mapred.IdentityTableReduce;
-import org.apache.hadoop.hbase.mapred.TableMap;
-import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.io.VectorUpdate;
-import org.apache.hama.util.BytesUtil;
-import org.apache.hama.util.JobManager;
-import org.apache.log4j.Logger;
-
-/**
- * Methods of the matrix classes
- */
-public abstract class AbstractMatrix implements Matrix {
- static final Logger LOG = Logger.getLogger(AbstractMatrix.class);
-
- protected HamaConfiguration config;
- protected HBaseAdmin admin;
- // a matrix just need a table path to point to the table which stores matrix.
- // let HamaAdmin manage Matrix Name space.
- protected String matrixPath;
- protected HTable table;
- protected HTableDescriptor tableDesc;
- protected HamaAdmin hamaAdmin;
-
- protected boolean closed = true;
-
- /**
- * Sets the job configuration
- *
- * @param conf configuration object
- * @throws MasterNotRunningException
- */
- public void setConfiguration(HamaConfiguration conf)
- throws MasterNotRunningException {
- this.config = conf;
- this.admin = new HBaseAdmin(config);
-
- hamaAdmin = new HamaAdminImpl(conf, admin);
- }
-
- /**
- * Create matrix space
- */
- protected void create() throws IOException {
- // It should run only when table doesn't exist.
- if (!admin.tableExists(matrixPath)) {
- this.tableDesc.addFamily(new HColumnDescriptor(Bytes
- .toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
- Integer.MAX_VALUE, HConstants.FOREVER, false));
- this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
- this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
- // It's a temporary data.
- this.tableDesc.addFamily(new HColumnDescriptor(Bytes
- .toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
- Integer.MAX_VALUE, HConstants.FOREVER, false));
-
- LOG.info("Initializing the matrix storage.");
- this.admin.createTable(this.tableDesc);
- LOG.info("Create Matrix " + matrixPath);
-
- // connect to the table.
- table = new HTable(config, matrixPath);
- table.setAutoFlush(true);
-
- // Record the matrix type in METADATA_TYPE
- BatchUpdate update = new BatchUpdate(Constants.METADATA);
- update.put(Constants.METADATA_TYPE, Bytes.toBytes(this.getClass()
- .getSimpleName()));
-
- table.commit(update);
-
- // the new matrix's reference is 1.
- setReference(1);
- }
- }
-
- public HTable getHTable() {
- return this.table;
- }
-
- /** {@inheritDoc} */
- public int getRows() throws IOException {
- Cell rows = null;
- rows = table.get(Constants.METADATA, Constants.METADATA_ROWS);
- return (rows != null) ? BytesUtil.bytesToInt(rows.getValue()) : 0;
- }
-
- /** {@inheritDoc} */
- public int getColumns() throws IOException {
- Cell columns = table.get(Constants.METADATA, Constants.METADATA_COLUMNS);
- return BytesUtil.bytesToInt(columns.getValue());
- }
-
- /** {@inheritDoc} */
- public String getRowLabel(int row) throws IOException {
- Cell rows = null;
- rows = table.get(BytesUtil.getRowIndex(row), Bytes
- .toBytes(Constants.ATTRIBUTE + "string"));
-
- return (rows != null) ? Bytes.toString(rows.getValue()) : null;
- }
-
- /** {@inheritDoc} */
- public String getColumnLabel(int column) throws IOException {
- Cell rows = null;
- rows = table.get(Constants.CINDEX, (Constants.ATTRIBUTE + column));
- return (rows != null) ? Bytes.toString(rows.getValue()) : null;
- }
-
- /** {@inheritDoc} */
- public void setRowLabel(int row, String name) throws IOException {
- VectorUpdate update = new VectorUpdate(row);
- update.put(Constants.ATTRIBUTE + "string", name);
- table.commit(update.getBatchUpdate());
- }
-
- /** {@inheritDoc} */
- public void set(int i, int j, double value) throws IOException {
- VectorUpdate update = new VectorUpdate(i);
- update.put(j, value);
- table.commit(update.getBatchUpdate());
-
- }
-
- /** {@inheritDoc} */
- public void setDimension(int rows, int columns) throws IOException {
- VectorUpdate update = new VectorUpdate(Constants.METADATA);
- update.put(Constants.METADATA_ROWS, rows);
- update.put(Constants.METADATA_COLUMNS, columns);
-
- table.commit(update.getBatchUpdate());
- }
-
- public void increaseRows() throws IOException {
- int nValue = this.getRows() + 1;
- try {
- this.setDimension(nValue, this.getColumns());
- } catch (NullPointerException ne) {
- // If there is no metadata of dimension, nothing to do.
- }
- }
-
- public void increaseColumns() throws IOException {
- int nValue = this.getColumns() + 1;
- try {
- this.setDimension(this.getRows(), nValue);
- } catch (NullPointerException ne) {
- // If there is no metadata of dimension, nothing to do.
- }
- }
-
- /** {@inheritDoc} */
- public void add(int i, int j, double value) throws IOException {
- VectorUpdate update = new VectorUpdate(i);
- update.put(j, value + this.get(i, j));
- table.commit(update.getBatchUpdate());
-
- }
-
- /**
- * Just full scan a table.
- */
- public static class TableReadMapper extends MapReduceBase implements
- TableMap<ImmutableBytesWritable, BatchUpdate> {
- private static List<Double> alpha = new ArrayList<Double>();
-
- @SuppressWarnings("unchecked")
- public void map(ImmutableBytesWritable key, RowResult value,
- OutputCollector<ImmutableBytesWritable, BatchUpdate> output,
- @SuppressWarnings("unused")
- Reporter reporter) throws IOException {
-
- BatchUpdate update = new BatchUpdate(key.get());
- for (Map.Entry<byte[], Cell> e : value.entrySet()) {
- if (alpha.size() == 0) {
- update.put(e.getKey(), e.getValue().getValue());
- } else {
- String column = new String(e.getKey());
- if(column.startsWith(Constants.COLUMN)) {
- double currValue = BytesUtil.bytesToDouble(e.getValue().getValue());
- update.put(e.getKey(), (BytesUtil.doubleToBytes(currValue * alpha.get(0))));
- } else {
- update.put(e.getKey(), e.getValue().getValue());
- }
- }
- }
- output.collect(key, update);
- }
-
- public static void setAlpha(double a) {
- if(alpha.size() > 0)
- alpha = new ArrayList<Double>();
- alpha.add(a);
- }
- }
-
- /** {@inheritDoc} */
- public Matrix set(Matrix B) throws IOException {
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("set MR job : " + this.getPath());
-
- jobConf.setNumMapTasks(config.getNumMapTasks());
- jobConf.setNumReduceTasks(config.getNumReduceTasks());
-
- TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
- + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
- + Constants.BLOCK, TableReadMapper.class, ImmutableBytesWritable.class,
- BatchUpdate.class, jobConf);
- TableMapReduceUtil.initTableReduceJob(this.getPath(),
- IdentityTableReduce.class, jobConf);
-
- JobManager.execute(jobConf);
- return this;
- }
-
- /** {@inheritDoc} */
- public Matrix set(double alpha, Matrix B) throws IOException {
- JobConf jobConf = new JobConf(config);
- jobConf.setJobName("set MR job : " + this.getPath());
-
- jobConf.setNumMapTasks(config.getNumMapTasks());
- jobConf.setNumReduceTasks(config.getNumReduceTasks());
-
- TableReadMapper.setAlpha(alpha);
- TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
- + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
- + Constants.BLOCK, TableReadMapper.class, ImmutableBytesWritable.class,
- BatchUpdate.class, jobConf);
- TableMapReduceUtil.initTableReduceJob(this.getPath(),
- IdentityTableReduce.class, jobConf);
-
- JobManager.execute(jobConf);
- return this;
- }
-
- /** {@inheritDoc} */
- public void setColumnLabel(int column, String name) throws IOException {
- VectorUpdate update = new VectorUpdate(Constants.CINDEX);
- update.put(column, name);
- table.commit(update.getBatchUpdate());
- }
-
- /** {@inheritDoc} */
- public String getPath() {
- return matrixPath;
- }
-
- protected void setReference(int reference) throws IOException {
- BatchUpdate update = new BatchUpdate(Constants.METADATA);
- update.put(Constants.METADATA_REFERENCE, Bytes.toBytes(reference));
- table.commit(update);
-
- }
-
- protected int incrementAndGetRef() throws IOException {
- int reference = 1;
- Cell rows = null;
- rows = table.get(Constants.METADATA, Constants.METADATA_REFERENCE);
- if (rows != null) {
- reference = Bytes.toInt(rows.getValue());
- reference++;
- }
- setReference(reference);
- return reference;
- }
-
- protected int decrementAndGetRef() throws IOException {
- int reference = 0;
- Cell rows = null;
- rows = table.get(Constants.METADATA, Constants.METADATA_REFERENCE);
- if (rows != null) {
- reference = Bytes.toInt(rows.getValue());
- if (reference > 0) // reference==0, we need not to decrement it.
- reference--;
- }
- setReference(reference);
- return reference;
- }
-
- protected boolean hasAliaseName() throws IOException {
- Cell rows = null;
- rows = table.get(Constants.METADATA, Constants.ALIASENAME);
- return (rows != null) ? true : false;
- }
-
- public void close() throws IOException {
- if (closed) // have been closed
- return;
- int reference = decrementAndGetRef();
- if (reference <= 0) { // no reference again.
- if (!hasAliaseName()) { // the table has not been aliased, we delete the
- // table.
- if (admin.isTableEnabled(matrixPath)) {
- while (admin.isTableEnabled(matrixPath)) {
- try {
- admin.disableTable(matrixPath);
- } catch (RegionException e) {
- LOG.warn(e);
- }
- }
-
- admin.deleteTable(matrixPath);
- }
- }
- }
- closed = true;
- }
-
- public boolean save(String aliasename) throws IOException {
- // mark & update the aliase name in "alise:name" meta column.
- // ! one matrix has only one aliasename now.
- BatchUpdate update = new BatchUpdate(Constants.METADATA);
- update.put(Constants.ALIASENAME, Bytes.toBytes(aliasename));
- table.commit(update);
-
- return hamaAdmin.save(this, aliasename);
- }
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.RegionException;
+import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.mapred.IdentityTableReduce;
+import org.apache.hadoop.hbase.mapred.TableMap;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorUpdate;
+import org.apache.hama.util.BytesUtil;
+import org.apache.hama.util.JobManager;
+import org.apache.hama.util.RandomVariable;
+import org.apache.log4j.Logger;
+
+/**
+ * Methods of the matrix classes
+ */
+public abstract class AbstractMatrix implements Matrix {
+ static int tryPathLength = Constants.DEFAULT_PATH_LENGTH;
+ static final String TABLE_PREFIX = DenseMatrix.class.getSimpleName() + "_";
+ static final Logger LOG = Logger.getLogger(AbstractMatrix.class);
+
+ protected HamaConfiguration config;
+ protected HBaseAdmin admin;
+ // a matrix just need a table path to point to the table which stores matrix.
+ // let HamaAdmin manage Matrix Name space.
+ protected String matrixPath;
+ protected HTable table;
+ protected HTableDescriptor tableDesc;
+ protected HamaAdmin hamaAdmin;
+
+ protected boolean closed = true;
+
+ /**
+ * Sets the job configuration
+ *
+ * @param conf configuration object
+ * @throws MasterNotRunningException
+ */
+ public void setConfiguration(HamaConfiguration conf)
+ throws MasterNotRunningException {
+ this.config = conf;
+ this.admin = new HBaseAdmin(config);
+
+ hamaAdmin = new HamaAdminImpl(conf, admin);
+ }
+
+ /**
+ * try to create a new matrix with a new random name. try times will be
+ * (Integer.MAX_VALUE - 4) * DEFAULT_TRY_TIMES;
+ *
+ * @throws IOException
+ */
+ protected void tryToCreateTable() throws IOException {
+ int tryTimes = Constants.DEFAULT_TRY_TIMES;
+ do {
+ matrixPath = TABLE_PREFIX + RandomVariable.randMatrixPath(tryPathLength);
+
+ if (!admin.tableExists(matrixPath)) { // no table 'matrixPath' in hbase.
+ tableDesc = new HTableDescriptor(matrixPath);
+ create();
+ return;
+ }
+
+ tryTimes--;
+ if (tryTimes <= 0) { // this loop has exhausted DEFAULT_TRY_TIMES.
+ tryPathLength++;
+ tryTimes = Constants.DEFAULT_TRY_TIMES;
+ }
+
+ } while (tryPathLength <= Constants.DEFAULT_MAXPATHLEN);
+ // exhaustes the try times.
+ // throw out an IOException to let the user know what happened.
+ throw new IOException("Try too many times to create a table in hbase.");
+ }
+
+ /**
+ * Create matrix space
+ */
+ protected void create() throws IOException {
+ // It should run only when table doesn't exist.
+ if (!admin.tableExists(matrixPath)) {
+ this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+ .toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
+ Integer.MAX_VALUE, HConstants.FOREVER, false));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
+ this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
+ // It's a temporary data.
+ this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+ .toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
+ Integer.MAX_VALUE, HConstants.FOREVER, false));
+
+ LOG.info("Initializing the matrix storage.");
+ this.admin.createTable(this.tableDesc);
+ LOG.info("Create Matrix " + matrixPath);
+
+ // connect to the table.
+ table = new HTable(config, matrixPath);
+ table.setAutoFlush(true);
+
+ // Record the matrix type in METADATA_TYPE
+ BatchUpdate update = new BatchUpdate(Constants.METADATA);
+ update.put(Constants.METADATA_TYPE, Bytes.toBytes(this.getClass()
+ .getSimpleName()));
+
+ table.commit(update);
+
+ // the new matrix's reference is 1.
+ setReference(1);
+ }
+ }
+
+ public HTable getHTable() {
+ return this.table;
+ }
+
+ /** {@inheritDoc} */
+ public int getRows() throws IOException {
+ Cell rows = null;
+ rows = table.get(Constants.METADATA, Constants.METADATA_ROWS);
+ return (rows != null) ? BytesUtil.bytesToInt(rows.getValue()) : 0;
+ }
+
+ /** {@inheritDoc} */
+ public int getColumns() throws IOException {
+ Cell columns = table.get(Constants.METADATA, Constants.METADATA_COLUMNS);
+ return BytesUtil.bytesToInt(columns.getValue());
+ }
+
+ /** {@inheritDoc} */
+ public String getRowLabel(int row) throws IOException {
+ Cell rows = null;
+ rows = table.get(BytesUtil.getRowIndex(row), Bytes
+ .toBytes(Constants.ATTRIBUTE + "string"));
+
+ return (rows != null) ? Bytes.toString(rows.getValue()) : null;
+ }
+
+ /** {@inheritDoc} */
+ public String getColumnLabel(int column) throws IOException {
+ Cell rows = null;
+ rows = table.get(Constants.CINDEX, (Constants.ATTRIBUTE + column));
+ return (rows != null) ? Bytes.toString(rows.getValue()) : null;
+ }
+
+ /** {@inheritDoc} */
+ public void setRowLabel(int row, String name) throws IOException {
+ VectorUpdate update = new VectorUpdate(row);
+ update.put(Constants.ATTRIBUTE + "string", name);
+ table.commit(update.getBatchUpdate());
+ }
+
+ /** {@inheritDoc} */
+ public void setDimension(int rows, int columns) throws IOException {
+ VectorUpdate update = new VectorUpdate(Constants.METADATA);
+ update.put(Constants.METADATA_ROWS, rows);
+ update.put(Constants.METADATA_COLUMNS, columns);
+
+ table.commit(update.getBatchUpdate());
+ }
+
+ public void increaseRows() throws IOException {
+ int nValue = this.getRows() + 1;
+ try {
+ this.setDimension(nValue, this.getColumns());
+ } catch (NullPointerException ne) {
+ // If there is no metadata of dimension, nothing to do.
+ }
+ }
+
+ public void increaseColumns() throws IOException {
+ int nValue = this.getColumns() + 1;
+ try {
+ this.setDimension(this.getRows(), nValue);
+ } catch (NullPointerException ne) {
+ // If there is no metadata of dimension, nothing to do.
+ }
+ }
+
+ /** {@inheritDoc} */
+ public void add(int i, int j, double value) throws IOException {
+ VectorUpdate update = new VectorUpdate(i);
+ update.put(j, value + this.get(i, j));
+ table.commit(update.getBatchUpdate());
+
+ }
+
+ /**
+ * Just full scan a table.
+ */
+ public static class TableReadMapper extends MapReduceBase implements
+ TableMap<ImmutableBytesWritable, BatchUpdate> {
+ private static List<Double> alpha = new ArrayList<Double>();
+
+ @SuppressWarnings("unchecked")
+ public void map(ImmutableBytesWritable key, RowResult value,
+ OutputCollector<ImmutableBytesWritable, BatchUpdate> output,
+ @SuppressWarnings("unused")
+ Reporter reporter) throws IOException {
+
+ BatchUpdate update = new BatchUpdate(key.get());
+ for (Map.Entry<byte[], Cell> e : value.entrySet()) {
+ if (alpha.size() == 0) {
+ update.put(e.getKey(), e.getValue().getValue());
+ } else {
+ String column = new String(e.getKey());
+ if(column.startsWith(Constants.COLUMN)) {
+ double currValue = BytesUtil.bytesToDouble(e.getValue().getValue());
+ update.put(e.getKey(), (BytesUtil.doubleToBytes(currValue * alpha.get(0))));
+ } else {
+ update.put(e.getKey(), e.getValue().getValue());
+ }
+ }
+ }
+ output.collect(key, update);
+ }
+
+ public static void setAlpha(double a) {
+ if(alpha.size() > 0)
+ alpha = new ArrayList<Double>();
+ alpha.add(a);
+ }
+ }
+
+ /** {@inheritDoc} */
+ public Matrix set(Matrix B) throws IOException {
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("set MR job : " + this.getPath());
+
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+ TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
+ + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
+ + Constants.BLOCK, TableReadMapper.class, ImmutableBytesWritable.class,
+ BatchUpdate.class, jobConf);
+ TableMapReduceUtil.initTableReduceJob(this.getPath(),
+ IdentityTableReduce.class, jobConf);
+
+ JobManager.execute(jobConf);
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ public Matrix set(double alpha, Matrix B) throws IOException {
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("set MR job : " + this.getPath());
+
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+ TableReadMapper.setAlpha(alpha);
+ TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
+ + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
+ + Constants.BLOCK, TableReadMapper.class, ImmutableBytesWritable.class,
+ BatchUpdate.class, jobConf);
+ TableMapReduceUtil.initTableReduceJob(this.getPath(),
+ IdentityTableReduce.class, jobConf);
+
+ JobManager.execute(jobConf);
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ public void setColumnLabel(int column, String name) throws IOException {
+ VectorUpdate update = new VectorUpdate(Constants.CINDEX);
+ update.put(column, name);
+ table.commit(update.getBatchUpdate());
+ }
+
+ /** {@inheritDoc} */
+ public String getPath() {
+ return matrixPath;
+ }
+
+ protected void setReference(int reference) throws IOException {
+ BatchUpdate update = new BatchUpdate(Constants.METADATA);
+ update.put(Constants.METADATA_REFERENCE, Bytes.toBytes(reference));
+ table.commit(update);
+
+ }
+
+ protected int incrementAndGetRef() throws IOException {
+ int reference = 1;
+ Cell rows = null;
+ rows = table.get(Constants.METADATA, Constants.METADATA_REFERENCE);
+ if (rows != null) {
+ reference = Bytes.toInt(rows.getValue());
+ reference++;
+ }
+ setReference(reference);
+ return reference;
+ }
+
+ protected int decrementAndGetRef() throws IOException {
+ int reference = 0;
+ Cell rows = null;
+ rows = table.get(Constants.METADATA, Constants.METADATA_REFERENCE);
+ if (rows != null) {
+ reference = Bytes.toInt(rows.getValue());
+ if (reference > 0) // reference==0, we need not to decrement it.
+ reference--;
+ }
+ setReference(reference);
+ return reference;
+ }
+
+ protected boolean hasAliaseName() throws IOException {
+ Cell rows = null;
+ rows = table.get(Constants.METADATA, Constants.ALIASENAME);
+ return (rows != null) ? true : false;
+ }
+
+ public void close() throws IOException {
+ if (closed) // have been closed
+ return;
+ int reference = decrementAndGetRef();
+ if (reference <= 0) { // no reference again.
+ if (!hasAliaseName()) { // the table has not been aliased, we delete the
+ // table.
+ if (admin.isTableEnabled(matrixPath)) {
+ while (admin.isTableEnabled(matrixPath)) {
+ try {
+ admin.disableTable(matrixPath);
+ } catch (RegionException e) {
+ LOG.warn(e);
+ }
+ }
+
+ admin.deleteTable(matrixPath);
+ }
+ }
+ }
+ closed = true;
+ }
+
+ public boolean save(String aliasename) throws IOException {
+ // mark & update the aliase name in "alise:name" meta column.
+ // ! one matrix has only one aliasename now.
+ BatchUpdate update = new BatchUpdate(Constants.METADATA);
+ update.put(Constants.ALIASENAME, Bytes.toBytes(aliasename));
+ table.commit(update);
+
+ return hamaAdmin.save(this, aliasename);
+ }
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java Tue Feb 24 02:52:48 2009
@@ -21,11 +21,9 @@
import java.util.Iterator;
-import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hama.io.DoubleEntry;
/**
* Methods of the vector classes
@@ -34,48 +32,6 @@
protected MapWritable entries;
/**
- * Gets the value of index
- *
- * @param index
- * @return the value of v(index)
- */
- public double get(int index) {
- double value;
- try {
- value = ((DoubleEntry) this.entries.get(new IntWritable(index))).getValue();
- } catch (NullPointerException e) {
- throw new ArrayIndexOutOfBoundsException(index);
- }
-
- return value;
- }
-
- /**
- * Sets the value of index
- *
- * @param index
- * @param value
- */
- public void set(int index, double value) {
- // If entries are null, create new object
- if(this.entries == null) {
- this.entries = new MapWritable();
- }
-
- this.entries.put(new IntWritable(index), new DoubleEntry(value));
- }
-
- /**
- * Adds the value to v(index)
- *
- * @param index
- * @param value
- */
- public void add(int index, double value) {
- set(index, get(index) + value);
- }
-
- /**
* Returns an Iterator.
*
* @return iterator
@@ -105,4 +61,11 @@
public MapWritable getEntries() {
return this.entries;
}
+
+ /**
+ * Clears the entries.
+ */
+ public void clear() {
+ this.entries = null;
+ }
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Feb 24 02:52:48 2009
@@ -25,7 +25,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scanner;
import org.apache.hadoop.hbase.io.Cell;
@@ -60,8 +59,6 @@
* This class represents a dense matrix.
*/
public class DenseMatrix extends AbstractMatrix implements Matrix {
- static int tryPathLength = Constants.DEFAULT_PATH_LENGTH;
- static final String TABLE_PREFIX = DenseMatrix.class.getSimpleName() + "_";
static private final Path TMP_DIR = new Path(DenseMatrix.class
.getSimpleName()
+ "_TMP_dir");
@@ -183,35 +180,6 @@
}
/**
- * try to create a new matrix with a new random name. try times will be
- * (Integer.MAX_VALUE - 4) * DEFAULT_TRY_TIMES;
- *
- * @throws IOException
- */
- private void tryToCreateTable() throws IOException {
- int tryTimes = Constants.DEFAULT_TRY_TIMES;
- do {
- matrixPath = TABLE_PREFIX + RandomVariable.randMatrixPath(tryPathLength);
-
- if (!admin.tableExists(matrixPath)) { // no table 'matrixPath' in hbase.
- tableDesc = new HTableDescriptor(matrixPath);
- create();
- return;
- }
-
- tryTimes--;
- if (tryTimes <= 0) { // this loop has exhausted DEFAULT_TRY_TIMES.
- tryPathLength++;
- tryTimes = Constants.DEFAULT_TRY_TIMES;
- }
-
- } while (tryPathLength <= Constants.DEFAULT_MAXPATHLEN);
- // exhaustes the try times.
- // throw out an IOException to let the user know what happened.
- throw new IOException("Try too many times to create a table in hbase.");
- }
-
- /**
* Generate matrix with random elements
*
* @param conf configuration object
@@ -310,9 +278,9 @@
* @return an m-by-n matrix with ones on the diagonal and zeros elsewhere.
* @throws IOException
*/
- public static Matrix identity(HamaConfiguration conf, int m, int n)
+ public static DenseMatrix identity(HamaConfiguration conf, int m, int n)
throws IOException {
- Matrix identity = new DenseMatrix(conf);
+ DenseMatrix identity = new DenseMatrix(conf);
LOG.info("Create the " + m + " * " + n + " identity matrix : "
+ identity.getPath());
@@ -341,7 +309,10 @@
throw new ArrayIndexOutOfBoundsException(i +", "+ j);
Cell c = table.get(BytesUtil.getRowIndex(i), BytesUtil.getColumnIndex(j));
- return (c != null) ? BytesUtil.bytesToDouble(c.getValue()) : 0;
+ if(c == null)
+ throw new NullPointerException("Unexpected null");
+
+ return BytesUtil.bytesToDouble(c.getValue());
}
/**
@@ -377,6 +348,13 @@
return new DenseVector(trunk);
}
+ /** {@inheritDoc} */
+ public void set(int i, int j, double value) throws IOException {
+ VectorUpdate update = new VectorUpdate(i);
+ update.put(j, value);
+ table.commit(update.getBatchUpdate());
+ }
+
/**
* Set the row of a matrix to a given vector
*
@@ -418,10 +396,10 @@
* @return C
* @throws IOException
*/
- public Matrix add(Matrix B) throws IOException {
+ public DenseMatrix add(Matrix B) throws IOException {
ensureForAddition(B);
- Matrix result = new DenseMatrix(config);
+ DenseMatrix result = new DenseMatrix(config);
JobConf jobConf = new JobConf(config);
jobConf.setJobName("addition MR job" + result.getPath());
@@ -446,12 +424,12 @@
* @return C
* @throws IOException
*/
- public Matrix add(double alpha, Matrix B) throws IOException {
+ public DenseMatrix add(double alpha, Matrix B) throws IOException {
ensureForAddition(B);
- Matrix temp = new DenseMatrix(config);
+ DenseMatrix temp = new DenseMatrix(config);
temp.set(alpha, B);
- Matrix result = this.add(temp);
+ DenseMatrix result = this.add(temp);
return result;
}
@@ -468,10 +446,10 @@
* @return C
* @throws IOException
*/
- public Matrix mult(Matrix B) throws IOException {
+ public DenseMatrix mult(Matrix B) throws IOException {
ensureForMultiplication(B);
- Matrix result = new DenseMatrix(config);
+ DenseMatrix result = new DenseMatrix(config);
JobConf jobConf = new JobConf(config);
jobConf.setJobName("multiplication MR job : " + result.getPath());
@@ -479,7 +457,7 @@
jobConf.setNumMapTasks(config.getNumMapTasks());
jobConf.setNumReduceTasks(config.getNumReduceTasks());
- SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), SIMDMultiplyMap.class,
+ SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), this.getType(), SIMDMultiplyMap.class,
IntWritable.class, MapWritable.class, jobConf);
SIMDMultiplyReduce.initJob(result.getPath(), SIMDMultiplyReduce.class,
jobConf);
@@ -495,16 +473,16 @@
* @return C
* @throws IOException
*/
- public Matrix mult(Matrix B, int blocks) throws IOException {
+ public DenseMatrix mult(Matrix B, int blocks) throws IOException {
ensureForMultiplication(B);
- Matrix collectionTable = new DenseMatrix(config);
+ DenseMatrix collectionTable = new DenseMatrix(config);
LOG.info("Collect Blocks");
collectBlocksMapRed(this.getPath(), collectionTable, blocks, true);
collectBlocksMapRed(B.getPath(), collectionTable, blocks, false);
- Matrix result = new DenseMatrix(config);
+ DenseMatrix result = new DenseMatrix(config);
JobConf jobConf = new JobConf(config);
jobConf.setJobName("multiplication MR job : " + result.getPath());
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Tue Feb 24 02:52:48 2009
@@ -19,6 +19,7 @@
*/
package org.apache.hama;
+import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
@@ -97,12 +98,11 @@
return this;
}
- for (int i = 0; i < this.size(); i++) {
- double value = (this.get(i) + v2.get(i));
-
- this.entries.put(new IntWritable(i), new DoubleEntry(value));
+ for(Map.Entry<Writable, Writable> e : this.getEntries().entrySet()) {
+ double value = ((DoubleEntry) e.getValue()).getValue() + v2.get(((IntWritable) e.getKey()).get());
+ this.entries.put(e.getKey(), new DoubleEntry(value));
}
-
+
return this;
}
@@ -129,7 +129,7 @@
* @param alpha
* @return v = alpha*v
*/
- public Vector scale(double alpha) {
+ public DenseVector scale(double alpha) {
for(Map.Entry<Writable, Writable> e : this.entries.entrySet()) {
this.entries.put(e.getKey(), new DoubleEntry(((DoubleEntry) e.getValue()).getValue() * alpha));
}
@@ -190,6 +190,49 @@
return Math.sqrt(square_sum);
}
+ /**
+ * Gets the value of index
+ *
+ * @param index
+ * @return the value of v(index)
+ * @throws IOException
+ */
+ public double get(int index) {
+ double value;
+ try {
+ value = ((DoubleEntry) this.entries.get(new IntWritable(index))).getValue();
+ } catch (NullPointerException e) {
+ throw new NullPointerException("Unexpected null value : " + e.toString());
+ }
+
+ return value;
+ }
+
+ /**
+ * Sets the value of index
+ *
+ * @param index
+ * @param value
+ */
+ public void set(int index, double value) {
+ // If entries are null, create new object
+ if(this.entries == null) {
+ this.entries = new MapWritable();
+ }
+
+ this.entries.put(new IntWritable(index), new DoubleEntry(value));
+ }
+
+ /**
+ * Adds the value to v(index)
+ *
+ * @param index
+ * @param value
+ */
+ public void add(int index, double value) {
+ set(index, get(index) + value);
+ }
+
public double getNorm2Robust() {
// TODO Auto-generated method stub
return 0;
@@ -218,11 +261,4 @@
return res;
}
-
- /**
- * Clears the entries.
- */
- public void clear() {
- this.entries = null;
- }
}
Added: incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java?rev=747265&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java Tue Feb 24 02:52:48 2009
@@ -0,0 +1,208 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hama.algebra.SIMDMultiplyMap;
+import org.apache.hama.algebra.SIMDMultiplyReduce;
+import org.apache.hama.io.VectorUpdate;
+import org.apache.hama.util.BytesUtil;
+import org.apache.hama.util.JobManager;
+import org.apache.hama.util.RandomVariable;
+
+public class SparseMatrix extends AbstractMatrix implements Matrix {
+
+ public SparseMatrix(HamaConfiguration conf) throws IOException {
+ setConfiguration(conf);
+
+ tryToCreateTable();
+
+ closed = false;
+ }
+
+ /**
+ * Load a matrix from an existed matrix table whose tablename is 'matrixpath' !!
+ * It is an internal used for map/reduce.
+ *
+ * @param conf configuration object
+ * @param matrixpath
+ * @throws IOException
+ * @throws IOException
+ */
+ public SparseMatrix(HamaConfiguration conf, String matrixpath)
+ throws IOException {
+ setConfiguration(conf);
+ matrixPath = matrixpath;
+ // load the matrix
+ table = new HTable(conf, matrixPath);
+ // TODO: now we don't increment the reference of the table
+ // for it's an internal use for map/reduce.
+ // if we want to increment the reference of the table,
+ // we don't know where to call Matrix.close in Add & Mul map/reduce
+ // process to decrement the reference. It seems difficulty.
+ }
+ /**
+ * Generate matrix with random elements
+ *
+ * @param conf configuration object
+ * @param m the number of rows.
+ * @param n the number of columns.
+ * @return an m-by-n matrix with uniformly distributed random elements.
+ * @throws IOException
+ */
+ public static SparseMatrix random(HamaConfiguration conf, int m, int n)
+ throws IOException {
+ SparseMatrix rand = new SparseMatrix(conf);
+ SparseVector vector = new SparseVector();
+ LOG.info("Create the " + m + " * " + n + " random matrix : "
+ + rand.getPath());
+
+ for (int i = 0; i < m; i++) {
+ vector.clear();
+ for (int j = 0; j < n; j++) {
+ Random r = new Random();
+ if(r.nextInt(2) != 0)
+ vector.set(j, RandomVariable.rand());
+ }
+ rand.setRow(i, vector);
+ }
+
+ rand.setDimension(m, n);
+ return rand;
+ }
+
+ @Override
+ public Matrix add(Matrix B) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Matrix add(double alpha, Matrix B) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public double get(int i, int j) throws IOException {
+ if(this.getRows() < i || this.getColumns() < j)
+ throw new ArrayIndexOutOfBoundsException(i +", "+ j);
+
+ Cell c = table.get(BytesUtil.getRowIndex(i), BytesUtil.getColumnIndex(j));
+ return (c != null) ? BytesUtil.bytesToDouble(c.getValue()) : 0.0;
+ }
+
+ @Override
+ public Vector getColumn(int j) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /**
+ * Gets the vector of row
+ *
+ * @param i the row index of the matrix
+ * @return the vector of row
+ * @throws IOException
+ */
+ public SparseVector getRow(int i) throws IOException {
+ // Should returns zero-fill vector.
+ return new SparseVector(table.getRow(BytesUtil.getRowIndex(i)));
+ }
+
+ /** {@inheritDoc} */
+ public void set(int i, int j, double value) throws IOException {
+ if(value != 0) {
+ VectorUpdate update = new VectorUpdate(i);
+ update.put(j, value);
+ table.commit(update.getBatchUpdate());
+ }
+ }
+
+ /**
+ * Returns type of matrix
+ */
+ public String getType() {
+ return this.getClass().getSimpleName();
+ }
+
+ @Override
+ public SparseMatrix mult(Matrix B) throws IOException {
+ SparseMatrix result = new SparseMatrix(config);
+
+ JobConf jobConf = new JobConf(config);
+ jobConf.setJobName("multiplication MR job : " + result.getPath());
+
+ jobConf.setNumMapTasks(config.getNumMapTasks());
+ jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+ SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), this.getType(), SIMDMultiplyMap.class,
+ IntWritable.class, MapWritable.class, jobConf);
+ SIMDMultiplyReduce.initJob(result.getPath(), SIMDMultiplyReduce.class,
+ jobConf);
+ JobManager.execute(jobConf);
+ result.setDimension(this.getRows(), this.getColumns());
+ return result;
+ }
+
+ @Override
+ public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public double norm(Norm type) throws IOException {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public void setColumn(int column, Vector vector) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void setRow(int row, Vector vector) throws IOException {
+ if(this.getRows() < row)
+ increaseRows();
+
+ if(vector.size() > 0) { // stores if size > 0
+ VectorUpdate update = new VectorUpdate(row);
+ update.putAll(((SparseVector) vector).getEntries());
+ table.commit(update.getBatchUpdate());
+ }
+ }
+
+ @Override
+ public SubMatrix subMatrix(int i0, int i1, int j0, int j1) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
Added: incubator/hama/trunk/src/java/org/apache/hama/SparseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SparseVector.java?rev=747265&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SparseVector.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/SparseVector.java Tue Feb 24 02:52:48 2009
@@ -0,0 +1,167 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.io.DoubleEntry;
+import org.apache.hama.util.BytesUtil;
+import org.apache.log4j.Logger;
+
+public class SparseVector extends AbstractVector implements Vector {
+ static final Logger LOG = Logger.getLogger(SparseVector.class);
+ public SparseVector() {
+ this(new MapWritable());
+ }
+
+ public SparseVector(MapWritable m) {
+ this.entries = m;
+ }
+
+ public SparseVector(RowResult row) {
+ this.entries = new MapWritable();
+ for (Map.Entry<byte[], Cell> f : row.entrySet()) {
+ this.entries.put(new IntWritable(BytesUtil.getColumnIndex(f.getKey())),
+ new DoubleEntry(f.getValue()));
+ }
+ }
+
+ @Override
+ public Vector add(double alpha, Vector v) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /**
+ * x = v + x
+ *
+ * @param v2
+ * @return x = v + x
+ */
+ public SparseVector add(Vector v2) {
+ if (this.size() == 0) {
+ SparseVector trunk = (SparseVector) v2;
+ this.entries = trunk.entries;
+ return this;
+ }
+
+ for(Map.Entry<Writable, Writable> e : ((SparseVector )v2).getEntries().entrySet()) {
+
+ if(this.entries.containsKey(e.getKey())) {
+ // add
+ double value = ((DoubleEntry) e.getValue()).getValue() + this.get(((IntWritable) e.getKey()).get());
+ this.entries.put(e.getKey(), new DoubleEntry(value));
+ } else {
+ // put
+ this.entries.put(e.getKey(), e.getValue());
+ }
+ }
+
+ return this;
+ }
+
+ @Override
+ public double dot(Vector v) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public double norm(Norm type) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ /**
+ * v = alpha*v
+ *
+ * @param alpha
+ * @return v = alpha*v
+ */
+ public SparseVector scale(double alpha) {
+ for(Map.Entry<Writable, Writable> e : this.entries.entrySet()) {
+ this.entries.put(e.getKey(), new DoubleEntry(((DoubleEntry) e.getValue()).getValue() * alpha));
+ }
+ return this;
+ }
+
+ /**
+ * Gets the value of index
+ *
+ * @param index
+ * @return the value of v(index)
+ * @throws IOException
+ */
+ public double get(int index) {
+ double value;
+ try {
+ value = ((DoubleEntry) this.entries.get(new IntWritable(index))).getValue();
+ } catch (NullPointerException e) { // returns zero if there is no value
+ return 0;
+ }
+
+ return value;
+ }
+
+ /**
+ * Sets the value of index
+ *
+ * @param index
+ * @param value
+ */
+ public void set(int index, double value) {
+ // If entries are null, create new object
+ if(this.entries == null) {
+ this.entries = new MapWritable();
+ }
+
+ if(value != 0) // only stores non-zero element
+ this.entries.put(new IntWritable(index), new DoubleEntry(value));
+ }
+
+ /**
+ * Adds the value to v(index)
+ *
+ * @param index
+ * @param value
+ */
+ public void add(int index, double value) {
+ set(index, get(index) + value);
+ }
+
+ @Override
+ public Vector set(Vector v) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Vector subVector(int i0, int i1) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
Modified: incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/SubMatrix.java Tue Feb 24 02:52:48 2009
@@ -1,223 +1,222 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.hama.util.BytesUtil;
-import org.apache.log4j.Logger;
-
-/**
- * A sub matrix is a matrix formed by selecting certain rows and columns from a
- * bigger matrix. This is a in-memory operation only.
- */
-public class SubMatrix {
- static final Logger LOG = Logger.getLogger(SubMatrix.class);
- private double[][] matrix;
-
- /**
- * Constructor
- *
- * @param i the size of rows
- * @param j the size of columns
- */
- public SubMatrix(int i, int j) {
- this.matrix = new double[i][j];
- }
-
- /**
- * Constructor
- *
- * @param c a two dimensional double array
- */
- public SubMatrix(double[][] c) {
- double[][] matrix = c;
- this.matrix = matrix;
- }
-
- public SubMatrix(byte[] matrix) throws IOException {
- ByteArrayInputStream bos = new ByteArrayInputStream(matrix);
- DataInputStream dis = new DataInputStream(bos);
-
- int rows = dis.readInt();
- int columns = dis.readInt();
- this.matrix = new double[rows][columns];
-
- for(int i = 0; i < rows; i++) {
- for(int j = 0; j < columns; j++) {
- this.matrix[i][j] = dis.readDouble();
- }
- }
-
- dis.close();
- bos.close();
- }
-
- /**
- * Sets the value
- *
- * @param row
- * @param column
- * @param value
- */
- public void set(int row, int column, double value) {
- matrix[row][column] = value;
- }
-
- /**
- * Sets the value
- *
- * @param row
- * @param column
- * @param value
- */
- public void set(int row, int column, byte[] value) {
- matrix[row][column] = BytesUtil.bytesToDouble(value);
- }
-
- /**
- * Gets the value
- *
- * @param i
- * @param j
- * @return the value of submatrix(i, j)
- */
- public double get(int i, int j) {
- return matrix[i][j];
- }
-
- public void add(int row, int column, double value) {
- matrix[row][column] = matrix[row][column] + value;
- }
-
- /**
- * c = a+b
- *
- * @param b
- * @return c
- */
- public SubMatrix add(SubMatrix b) {
- SubMatrix c = new SubMatrix(this.getRows(), this.getColumns());
-
- for (int i = 0; i < this.getRows(); i++) {
- for (int j = 0; j < this.getColumns(); j++) {
- c.set(i, j, (this.get(i, j) + b.get(i, j)));
- }
- }
-
- return c;
- }
-
- /**
- * c = a*b
- *
- * @param b
- * @return c
- */
- public SubMatrix mult(SubMatrix b) {
- SubMatrix c = new SubMatrix(this.getRows(), b.getColumns());
-
- for (int i = 0; i < this.getRows(); i++) {
- for (int j = 0; j < b.getColumns(); j++) {
- for (int k = 0; k < this.getColumns(); k++) {
- c.add(i, j, this.get(i, k) * b.get(k, j));
- }
- }
- }
-
- return c;
- }
-
- /**
- * Gets the number of rows
- *
- * @return the number of rows
- */
- public int getRows() {
- return this.matrix.length;
- }
-
- /**
- * Gets the number of columns
- *
- * @return the number of columns
- */
- public int getColumns() {
- return this.matrix[0].length;
- }
-
- /**
- * Close
- */
- public void close() {
- matrix = null;
- }
-
- /**
- * @return the 2d double array
- */
- public double[][] getDoubleArray() {
- double[][] result = matrix;
- return result;
- }
-
- /**
- * Gets the bytes of the sub matrix
- *
- * @return the bytes of the sub matrix
- * @throws IOException
- */
- public byte[] getBytes() throws IOException {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(bos);
-
- dos.writeInt(this.getRows());
- dos.writeInt(this.getColumns());
-
- for(int i = 0; i < this.getRows(); i++) {
- for(int j = 0; j < this.getColumns(); j++) {
- dos.writeDouble(this.get(i, j));
- }
- }
-
- byte[] data = bos.toByteArray();
- dos.close();
- bos.close();
- return data;
- }
-
- public String toString() {
- StringBuilder result = new StringBuilder();
- for (int i = 0; i < this.getRows(); i++) {
- for (int j = 0; j < this.getColumns(); j++) {
- result.append(this.get(i, j));
- result.append('\t');
- }
- result.append('\n');
- }
- return result.toString();
- }
-}
-
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hama.util.BytesUtil;
+import org.apache.log4j.Logger;
+
+/**
+ * A sub matrix is a matrix formed by selecting certain rows and columns from a
+ * bigger matrix. This is a in-memory operation only.
+ */
+public class SubMatrix {
+ static final Logger LOG = Logger.getLogger(SubMatrix.class);
+ private double[][] matrix;
+
+ /**
+ * Constructor
+ *
+ * @param i the size of rows
+ * @param j the size of columns
+ */
+ public SubMatrix(int i, int j) {
+ this.matrix = new double[i][j];
+ }
+
+ /**
+ * Constructor
+ *
+ * @param c a two dimensional double array
+ */
+ public SubMatrix(double[][] c) {
+ double[][] matrix = c;
+ this.matrix = matrix;
+ }
+
+ public SubMatrix(byte[] matrix) throws IOException {
+ ByteArrayInputStream bos = new ByteArrayInputStream(matrix);
+ DataInputStream dis = new DataInputStream(bos);
+
+ int rows = dis.readInt();
+ int columns = dis.readInt();
+ this.matrix = new double[rows][columns];
+
+ for(int i = 0; i < rows; i++) {
+ for(int j = 0; j < columns; j++) {
+ this.matrix[i][j] = dis.readDouble();
+ }
+ }
+
+ dis.close();
+ bos.close();
+ }
+
+ /**
+ * Sets the value
+ *
+ * @param row
+ * @param column
+ * @param value
+ */
+ public void set(int row, int column, double value) {
+ matrix[row][column] = value;
+ }
+
+ /**
+ * Sets the value
+ *
+ * @param row
+ * @param column
+ * @param value
+ */
+ public void set(int row, int column, byte[] value) {
+ matrix[row][column] = BytesUtil.bytesToDouble(value);
+ }
+
+ /**
+ * Gets the value
+ *
+ * @param i
+ * @param j
+ * @return the value of submatrix(i, j)
+ */
+ public double get(int i, int j) {
+ return matrix[i][j];
+ }
+
+ public void add(int row, int column, double value) {
+ matrix[row][column] = matrix[row][column] + value;
+ }
+
+ /**
+ * c = a+b
+ *
+ * @param b
+ * @return c
+ */
+ public SubMatrix add(SubMatrix b) {
+ SubMatrix c = new SubMatrix(this.getRows(), this.getColumns());
+
+ for (int i = 0; i < this.getRows(); i++) {
+ for (int j = 0; j < this.getColumns(); j++) {
+ c.set(i, j, (this.get(i, j) + b.get(i, j)));
+ }
+ }
+
+ return c;
+ }
+
+ /**
+ * c = a*b
+ *
+ * @param b
+ * @return c
+ */
+ public SubMatrix mult(SubMatrix b) {
+ SubMatrix c = new SubMatrix(this.getRows(), b.getColumns());
+
+ for (int i = 0; i < this.getRows(); i++) {
+ for (int j = 0; j < b.getColumns(); j++) {
+ for (int k = 0; k < this.getColumns(); k++) {
+ c.add(i, j, this.get(i, k) * b.get(k, j));
+ }
+ }
+ }
+
+ return c;
+ }
+
+ /**
+ * Gets the number of rows
+ *
+ * @return the number of rows
+ */
+ public int getRows() {
+ return this.matrix.length;
+ }
+
+ /**
+ * Gets the number of columns
+ *
+ * @return the number of columns
+ */
+ public int getColumns() {
+ return this.matrix[0].length;
+ }
+
+ /**
+ * Close
+ */
+ public void close() {
+ matrix = null;
+ }
+
+ /**
+ * @return the 2d double array
+ */
+ public double[][] getDoubleArray() {
+ double[][] result = matrix;
+ return result;
+ }
+
+ /**
+ * Gets the bytes of the sub matrix
+ *
+ * @return the bytes of the sub matrix
+ * @throws IOException
+ */
+ public byte[] getBytes() throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(bos);
+
+ dos.writeInt(this.getRows());
+ dos.writeInt(this.getColumns());
+
+ for(int i = 0; i < this.getRows(); i++) {
+ for(int j = 0; j < this.getColumns(); j++) {
+ dos.writeDouble(this.get(i, j));
+ }
+ }
+
+ byte[] data = bos.toByteArray();
+ dos.close();
+ bos.close();
+ return data;
+ }
+
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < this.getRows(); i++) {
+ for (int j = 0; j < this.getColumns(); j++) {
+ result.append(this.get(i, j));
+ result.append('\t');
+ }
+ result.append('\n');
+ }
+ return result.toString();
+ }
+}
+
Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Tue Feb 24 02:52:48 2009
@@ -34,7 +34,9 @@
import org.apache.hama.DenseVector;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.Matrix;
-import org.apache.hama.io.DoubleEntry;
+import org.apache.hama.SparseMatrix;
+import org.apache.hama.SparseVector;
+import org.apache.hama.Vector;
import org.apache.hama.mapred.VectorInputFormat;
import org.apache.log4j.Logger;
@@ -45,18 +47,27 @@
Mapper<IntWritable, MapWritable, IntWritable, MapWritable> {
static final Logger LOG = Logger.getLogger(SIMDMultiplyMap.class);
protected Matrix matrix_b;
+ protected Vector sum;
+ protected String type;
public static final String MATRIX_B = "hama.multiplication.matrix.b";
- public static final DenseVector sum = new DenseVector();;
+ public static final String MATRIX_TYPE = "hama.multiplication.matrix.type";
public void configure(JobConf job) {
try {
- matrix_b = new DenseMatrix(new HamaConfiguration(job), job.get(MATRIX_B, ""));
+ type = job.get(MATRIX_TYPE, "");
+ if(type.equals("SparseMatrix")) {
+ matrix_b = new SparseMatrix(new HamaConfiguration(job), job.get(MATRIX_B, ""));
+ sum = new SparseVector();
+ } else {
+ matrix_b = new DenseMatrix(new HamaConfiguration(job), job.get(MATRIX_B, ""));
+ sum = new DenseVector();
+ }
} catch (IOException e) {
LOG.warn("Load matrix_b failed : " + e.getMessage());
}
}
- public static void initJob(String matrix_a, String matrix_b,
+ public static void initJob(String matrix_a, String matrix_b, String type,
Class<SIMDMultiplyMap> map, Class<IntWritable> outputKeyClass,
Class<MapWritable> outputValueClass, JobConf jobConf) {
@@ -64,6 +75,7 @@
jobConf.setMapOutputKeyClass(outputKeyClass);
jobConf.setMapperClass(map);
jobConf.set(MATRIX_B, matrix_b);
+ jobConf.set(MATRIX_TYPE, type);
jobConf.setInputFormat(VectorInputFormat.class);
FileInputFormat.addInputPaths(jobConf, matrix_a);
@@ -74,12 +86,26 @@
public void map(IntWritable key, MapWritable value,
OutputCollector<IntWritable, MapWritable> output, Reporter reporter)
throws IOException {
- sum.clear();
-
- for(int i = 0; i < value.size(); i++) {
- sum.add(matrix_b.getRow(i).scale(((DoubleEntry) value.get(new IntWritable(i))).getValue()));
+ if(type.equals("SparseMatrix")) {
+ ((SparseVector) sum).clear();
+ SparseVector currVector = new SparseVector(value);
+
+ for(int i = 0; i < matrix_b.getColumns(); i++) {
+ ((SparseVector) sum).add(((SparseMatrix) matrix_b).getRow(i).scale(
+ currVector.get(i)));
+ }
+
+
+ output.collect(key, ((SparseVector) sum).getEntries());
+ } else {
+ ((DenseVector) sum).clear();
+ DenseVector currVector = new DenseVector(value);
+ for(int i = 0; i < value.size(); i++) {
+ ((DenseVector) sum).add(((DenseMatrix) matrix_b).getRow(i).scale(
+ currVector.get(i)));
+ }
+ output.collect(key, ((DenseVector) sum).getEntries());
}
- output.collect(key, sum.getEntries());
}
}
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Tue Feb 24 02:52:48 2009
@@ -185,8 +185,9 @@
a.set(m1);
for (int i = 0; i < 5; i++) {
- int x = RandomVariable.randInt(0, 10);
- int y = RandomVariable.randInt(0, 10);
+ // between 0 ~ SIZE -1
+ int x = RandomVariable.randInt(0, SIZE - 1);
+ int y = RandomVariable.randInt(0, SIZE - 1);
assertEquals(a.get(x, y), m1.get(x, y));
}
}
@@ -196,8 +197,8 @@
a.set(0.5, m1);
for (int i = 0; i < 5; i++) {
- int x = RandomVariable.randInt(0, 10);
- int y = RandomVariable.randInt(0, 10);
+ int x = RandomVariable.randInt(0, SIZE - 1);
+ int y = RandomVariable.randInt(0, SIZE - 1);
assertEquals(a.get(x, y), (m1.get(x, y) * 0.5));
}
}
@@ -274,8 +275,8 @@
// load matrix m1 using aliase1
DenseMatrix loadTest = new DenseMatrix(conf, aliase1, false);
- for (int i = 0; i < loadTest.getRows(); i++) {
- for (int j = 0; j < loadTest.getColumns(); j++) {
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
assertEquals(m1.get(i, j), loadTest.get(i, j));
}
}
Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java?rev=747265&r1=747264&r2=747265&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java Tue Feb 24 02:52:48 2009
@@ -39,9 +39,9 @@
private static final double norm1 = 12.0;
private static final double norm2 = 6.782329983125268;
private static double[][] values = { { 2, 5, 1, 4 }, { 4, 1, 3, 3 } };
- private static Matrix m1;
- private static Vector v1;
- private static Vector v2;
+ private static DenseMatrix m1;
+ private static DenseVector v1;
+ private static DenseVector v2;
public static Test suite() {
TestSetup setup = new TestSetup(new TestSuite(TestDenseVector.class)) {
@@ -122,7 +122,7 @@
boolean ex = false;
try {
v1.get(5);
- } catch (ArrayIndexOutOfBoundsException e) {
+ } catch (NullPointerException e) {
ex = true;
}
assertTrue(ex);
Added: incubator/hama/trunk/src/test/org/apache/hama/TestSparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestSparseMatrix.java?rev=747265&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestSparseMatrix.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestSparseMatrix.java Tue Feb 24 02:52:48 2009
@@ -0,0 +1,112 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+
+import junit.extensions.TestSetup;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+import org.apache.log4j.Logger;
+
+public class TestSparseMatrix extends TestCase {
+ static final Logger LOG = Logger.getLogger(TestSparseMatrix.class);
+ private static int SIZE = 10;
+ private static SparseMatrix m1;
+ private static SparseMatrix m2;
+
+ public static Test suite() {
+ TestSetup setup = new TestSetup(new TestSuite(TestSparseMatrix.class)) {
+ protected void setUp() throws Exception {
+ HCluster hCluster = new HCluster();
+ hCluster.setUp();
+
+ m1 = SparseMatrix.random(hCluster.getConf(), SIZE, SIZE);
+ m2 = SparseMatrix.random(hCluster.getConf(), SIZE, SIZE);
+ }
+
+ protected void tearDown() {
+ try {
+ closeTest();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ };
+ return setup;
+ }
+
+ public static void closeTest() throws IOException {
+ m1.close();
+ m2.close();
+ }
+
+ public void testSparsity() throws IOException {
+ boolean appeared = false;
+ for(int i = 0; i < m1.getRows(); i++) {
+ for(int j = 0; j < m1.getColumns(); j++) {
+ if(m1.get(i, j) == 0)
+ appeared = true;
+ }
+ }
+
+ assertTrue(appeared);
+ }
+
+ /**
+ * Test matrices multiplication
+ *
+ * @throws IOException
+ */
+ public void testMatrixMult() throws IOException {
+ SparseMatrix result = m1.mult(m2);
+ verifyMultResult(m1, m2, result);
+ }
+
+ /**
+ * Verifying multiplication result
+ *
+ * @param m1
+ * @param m2
+ * @param result
+ * @throws IOException
+ */
+ private void verifyMultResult(SparseMatrix m1, SparseMatrix m2, SparseMatrix result)
+ throws IOException {
+ double[][] c = new double[SIZE][SIZE];
+
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
+ for (int k = 0; k < SIZE; k++) {
+ c[i][k] += m1.get(i, j) * m2.get(j, k);
+ }
+ }
+ }
+
+ for (int i = 0; i < SIZE; i++) {
+ for (int j = 0; j < SIZE; j++) {
+ double gap = (c[i][j] - result.get(i, j));
+ assertTrue(gap < 0.000001 || gap < -0.000001);
+ }
+ }
+ }
+}
Added: incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java?rev=747265&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java Tue Feb 24 02:52:48 2009
@@ -0,0 +1,77 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+
+import junit.extensions.TestSetup;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hama.util.BytesUtil;
+
+public class TestSparseVector extends TestCase {
+ final static Log LOG = LogFactory.getLog(TestSparseVector.class.getName());
+ private static SparseMatrix m1;
+ private static SparseVector v1;
+ private static SparseVector v2;
+ private static double[][] values = { { 2, 0, 0, 4 }, { 0, 0, 3, 3 } };
+
+ public static Test suite() {
+ TestSetup setup = new TestSetup(new TestSuite(TestSparseVector.class)) {
+ protected void setUp() throws Exception {
+ HCluster hCluster = new HCluster();
+ hCluster.setUp();
+
+ m1 = new SparseMatrix(hCluster.getConf());
+
+ for (int i = 0; i < 2; i++)
+ for (int j = 0; j < 4; j++)
+ m1.set(i, j, values[i][j]);
+
+ v1 = m1.getRow(0);
+ v2 = m1.getRow(1);
+ }
+
+ protected void tearDown() {
+ LOG.info("tearDown()");
+ }
+ };
+ return setup;
+ }
+
+ /**
+ * Test get/set methods
+ * @throws IOException
+ */
+ public void testGetSet() throws IOException {
+ assertEquals(v1.get(1), 0.0);
+ assertEquals(v2.get(1), 0.0);
+
+ HTable table = m1.getHTable();
+ Cell c = table.get(BytesUtil.getRowIndex(0), BytesUtil.getColumnIndex(1));
+ assertTrue(c == null);
+ }
+}